<commit_before>// Package vat provides VAT number verification for Golang.
package vat

import (
	"bytes"
	"encoding/xml"
	"errors"
	"html/template"
	"io/ioutil"
	"net/http"
	"regexp"
	"strings"
	"time"
)

type viesResponse struct {
	CountryCode string
	VATnumber   string
	RequestDate time.Time
	Valid       bool
	Name        string
	Address     string
}

const serviceURL = "http://ec.europa.eu/taxation_customs/vies/services/checkVatService"
const envelope = `
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:v1="http://schemas.conversesolutions.com/xsd/dmticta/v1">
<soapenv:Header/>
<soapenv:Body>
  <checkVat xmlns="urn:ec.europa.eu:taxud:vies:services:checkVat:types">
    <countryCode>{{.countryCode}}</countryCode>
    <vatNumber>{{.vatNumber}}</vatNumber>
  </checkVat>
</soapenv:Body>
</soapenv:Envelope>
`

var (
	ErrInvalidVATNumber   = errors.New("VAT number is invalid.")
	ErrServiceUnreachable = errors.New("Validation service is offline.")
)

// Validate validates a VAT number by format and existence.
//
// The existence check uses the VIES VAT validation SOAP API and will only run when format validation passes.
func Validate(n string) (bool, error) {
	format, err := ValidateFormat(n)
	existence := false

	if format {
		existence, err = ValidateExistence(n)
	}

	return (format && existence), err
}

// ValidateFormat validates a VAT number by its format.
func ValidateFormat(n string) (bool, error) {
	patterns := map[string]string{
		"AT": "U[A-Z\\d]{8}",
		"BE": "(0\\d{9}|\\d{10})",
		"BG": "\\d{9,10}",
		"CY": "\\d{8}[A-Z]",
		"CZ": "\\d{8,10}",
		"DE": "\\d{9}",
		"DK": "(\\d{2} ?){3}\\d{2}",
		"EE": "\\d{9}",
		"EL": "\\d{9}",
		"ES": "[A-Z]\\d{7}[A-Z]|\\d{8}[A-Z]|[A-Z]\\d{8}",
		"FI": "\\d{8}",
		"FR": "([A-Z]{2}|\\d{2})\\d{9}",
		"GB": "\\d{9}|\\d{12}|(GD|HA)\\d{3}",
		"HR": "\\d{11}",
		"HU": "\\d{8}",
		"IE": "[A-Z\\d]{8}|[A-Z\\d]{9}",
		"IT": "\\d{11}",
		"LT": "(\\d{9}|\\d{12})",
		"LU": "\\d{8}",
		"LV": "\\d{11}",
		"MT": "\\d{8}",
		"NL": "\\d{9}B\\d{2}",
		"PL": "\\d{10}",
		"PT": "\\d{9}",
		"RO": "\\d{2,10}",
		"SE": "\\d{12}",
		"SI": "\\d{8}",
		"SK": "\\d{10}",
	}

	if len(n) < 3 {
		return false, nil
	}

	n = strings.ToUpper(n)
	pattern, ok := patterns[n[0:2]]
	if !ok {
		return false, nil
	}

	matched, err := regexp.MatchString(pattern, n[2:])
	return matched, err
}

// ValidateExistence validates a VAT number by its existence using the VIES VAT API (using SOAP)
func ValidateExistence(n string) (bool, error) {
	r, err := checkVAT(n)
	return r.Valid, err
}

// Check returns *VATresponse for vat number
func checkVAT(vatNumber string) (*viesResponse, error) {
	if len(vatNumber) < 3 {
		return nil, ErrInvalidVATNumber
	}

	e, err := getEnvelope(vatNumber)
	if err != nil {
		return nil, err
	}
	eb := bytes.NewBufferString(e)
	client := http.Client{
		Timeout: 10 * time.Second,
	}
	res, err := client.Post(serviceURL, "text/xml;charset=UTF-8", eb)
	if err != nil {
		return nil, ErrServiceUnreachable
	}
	defer res.Body.Close()

	// TODO: Use reader XML decoder
	xmlRes, err := ioutil.ReadAll(res.Body)

	// check if response contains "INVALID_INPUT" string
	if bytes.Contains(xmlRes, []byte("INVALID_INPUT")) {
		return nil, ErrInvalidVATNumber
	}

	var rd struct {
		XMLName xml.Name `xml:"Envelope"`
		Soap    struct {
			XMLName xml.Name `xml:"Body"`
			Soap    struct {
				XMLName     xml.Name `xml:"checkVatResponse"`
				CountryCode string   `xml:"countryCode"`
				VATnumber   string   `xml:"vatNumber"`
				RequestDate string   `xml:"requestDate"` // 2015-03-06+01:00
				Valid       bool     `xml:"valid"`
				Name        string   `xml:"name"`
				Address     string   `xml:"address"`
			}
		}
	}
	if err := xml.Unmarshal(xmlRes, &rd); err != nil {
		return nil, err
	}

	pDate, err := time.Parse("2006-01-02-07:00", rd.Soap.Soap.RequestDate)
	if err != nil {
		return nil, err
	}

	r := &viesResponse{
		CountryCode: rd.Soap.Soap.CountryCode,
		VATnumber:   rd.Soap.Soap.VATnumber,
		RequestDate: pDate,
		Valid:       rd.Soap.Soap.Valid,
		Name:        rd.Soap.Soap.Name,
		Address:     rd.Soap.Soap.Address,
	}

	return r, nil
}

// getEnvelope parses envelope template
func getEnvelope(vatNumber string) (string, error) {
	t, err := template.New("envelope").Parse(envelope)
	if err != nil {
		return "", err
	}

	var result bytes.Buffer
	if err := t.Execute(&result, map[string]string{
		"countryCode": strings.ToUpper(vatNumber[0:2]),
		"vatNumber":   vatNumber[2:],
	}); err != nil {
		return "", err
	}
	return result.String(), nil
}
<commit_msg>Stop using `html/template` for simple string replacement and improve godoc.<commit_after>/*
Package vat provides VAT number verification for Golang.

Example:
		// validates format + existence
		validity := vat.Validate("NL123456789B01")

		// validate format
		validity := vat.ValidateFormat("NL123456789B01")

		// validate existence
		validity := vat.ValidateExistence("NL123456789B01")
*/
package vat

import (
	"bytes"
	"encoding/xml"
	"errors"
	"io/ioutil"
	"net/http"
	"regexp"
	"strings"
	"time"
)

type viesResponse struct {
	CountryCode string
	VATNumber   string
	RequestDate time.Time
	Valid       bool
	Name        string
	Address     string
}

const serviceURL = "http://ec.europa.eu/taxation_customs/vies/services/checkVatService"

// ErrInvalidVATNumber will be returned when an invalid VAT number is passed to a function that validates existence.
var ErrInvalidVATNumber = errors.New("VAT number is invalid")

// ErrServiceUnreachable will be returned when VIES VAT validation API is unreachable.
var ErrServiceUnreachable = errors.New("Validation service is offline")

// Validate validates a VAT number by both format and existence.
// The existence check uses the VIES VAT validation SOAP API and will only run when format validation passes.
func Validate(n string) (bool, error) {
	format, err := ValidateFormat(n)
	existence := false

	if format {
		existence, err = ValidateExistence(n)
	}

	return (format && existence), err
}

// ValidateFormat validates a VAT number by its format.
func ValidateFormat(n string) (bool, error) {
	patterns := map[string]string{
		"AT": "U[A-Z\\d]{8}",
		"BE": "(0\\d{9}|\\d{10})",
		"BG": "\\d{9,10}",
		"CY": "\\d{8}[A-Z]",
		"CZ": "\\d{8,10}",
		"DE": "\\d{9}",
		"DK": "(\\d{2} ?){3}\\d{2}",
		"EE": "\\d{9}",
		"EL": "\\d{9}",
		"ES": "[A-Z]\\d{7}[A-Z]|\\d{8}[A-Z]|[A-Z]\\d{8}",
		"FI": "\\d{8}",
		"FR": "([A-Z]{2}|\\d{2})\\d{9}",
		"GB": "\\d{9}|\\d{12}|(GD|HA)\\d{3}",
		"HR": "\\d{11}",
		"HU": "\\d{8}",
		"IE": "[A-Z\\d]{8}|[A-Z\\d]{9}",
		"IT": "\\d{11}",
		"LT": "(\\d{9}|\\d{12})",
		"LU": "\\d{8}",
		"LV": "\\d{11}",
		"MT": "\\d{8}",
		"NL": "\\d{9}B\\d{2}",
		"PL": "\\d{10}",
		"PT": "\\d{9}",
		"RO": "\\d{2,10}",
		"SE": "\\d{12}",
		"SI": "\\d{8}",
		"SK": "\\d{10}",
	}

	if len(n) < 3 {
		return false, nil
	}

	n = strings.ToUpper(n)
	pattern, ok := patterns[n[0:2]]
	if !ok {
		return false, nil
	}

	matched, err := regexp.MatchString(pattern, n[2:])
	return matched, err
}

// ValidateExistence validates a VAT number by its existence using the VIES VAT API (using SOAP)
func ValidateExistence(n string) (bool, error) {
	r, err := checkVAT(n)
	return r.Valid, err
}

// checkVAT returns *ViesResponse for a VAT number
func checkVAT(vatNumber string) (*viesResponse, error) {
	if len(vatNumber) < 3 {
		return nil, ErrInvalidVATNumber
	}

	e := getEnvelope(vatNumber)
	eb := bytes.NewBufferString(e)
	client := http.Client{
		Timeout: 10 * time.Second,
	}
	res, err := client.Post(serviceURL, "text/xml;charset=UTF-8", eb)
	if err != nil {
		return nil, ErrServiceUnreachable
	}
	defer res.Body.Close()

	xmlRes, err := ioutil.ReadAll(res.Body)

	// check if response contains "INVALID_INPUT" string
	if bytes.Contains(xmlRes, []byte("INVALID_INPUT")) {
		return nil, ErrInvalidVATNumber
	}

	var rd struct {
		XMLName xml.Name `xml:"Envelope"`
		Soap    struct {
			XMLName xml.Name `xml:"Body"`
			Soap    struct {
				XMLName     xml.Name `xml:"checkVatResponse"`
				CountryCode string   `xml:"countryCode"`
				VATNumber   string   `xml:"vatNumber"`
				RequestDate string   `xml:"requestDate"` // 2015-03-06+01:00
				Valid       bool     `xml:"valid"`
				Name        string   `xml:"name"`
				Address     string   `xml:"address"`
			}
		}
	}
	if err := xml.Unmarshal(xmlRes, &rd); err != nil {
		return nil, err
	}

	pDate, err := time.Parse("2006-01-02-07:00", rd.Soap.Soap.RequestDate)
	if err != nil {
		return nil, err
	}

	r := &viesResponse{
		CountryCode: rd.Soap.Soap.CountryCode,
		VATNumber:   rd.Soap.Soap.VATNumber,
		RequestDate: pDate,
		Valid:       rd.Soap.Soap.Valid,
		Name:        rd.Soap.Soap.Name,
		Address:     rd.Soap.Soap.Address,
	}

	return r, nil
}

// getEnvelope parses envelope template
func getEnvelope(n string) string {
	n = strings.ToUpper(n)
	countryCode := n[0:2]
	vatNumber := n[2:]
	const envelopeTemplate = `
	<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/">
	<soapenv:Header/>
	<soapenv:Body>
	  <checkVat xmlns="urn:ec.europa.eu:taxud:vies:services:checkVat:types">
	    <countryCode>{{.countryCode}}</countryCode>
	    <vatNumber>{{.vatNumber}}</vatNumber>
	  </checkVat>
	</soapenv:Body>
	</soapenv:Envelope>
	`

	e := envelopeTemplate
	e = strings.Replace(e, "{{.countryCode}}", countryCode, 1)
	e = strings.Replace(e, "{{.vatNumber}}", vatNumber, 1)
	return e
}
<|endoftext|>


<commit_before>/*
Package vat helps you deal with European VAT in Go.

It offers VAT number validation using the VIES VAT validation API & VAT rates retrieval using jsonvat.com

Validate a VAT number
		validity := vat.ValidateNumber("NL123456789B01")

Get VAT rate that is currently in effect for a given country
		c, _ := vat.GetCountryRates("NL")
		r, _ := c.GetRate("standard")
*/
package vat

import "errors"

// ErrServiceUnavailable will be returned when VIES VAT validation API or jsonvat.com is unreachable.
var ErrServiceUnavailable = errors.New("Service is unreachable")

// ServiceTimeout indicates the number of seconds before a service request times out.
var ServiceTimeout = 10
<commit_msg>lowercase error string & prefix with package name<commit_after>/*
Package vat helps you deal with European VAT in Go.

It offers VAT number validation using the VIES VAT validation API & VAT rates retrieval using jsonvat.com

Validate a VAT number
		validity := vat.ValidateNumber("NL123456789B01")

Get VAT rate that is currently in effect for a given country
		c, _ := vat.GetCountryRates("NL")
		r, _ := c.GetRate("standard")
*/
package vat

import "errors"

// ErrServiceUnavailable will be returned when VIES VAT validation API or jsonvat.com is unreachable.
var ErrServiceUnavailable = errors.New("vat: service is unreachable")

// ServiceTimeout indicates the number of seconds before a service request times out.
var ServiceTimeout = 10
<|endoftext|>


<commit_before>package vox

// #cgo LDFLAGS: -lstdc++ -ldl -lm
// #define SUNVOX_MAIN
// #include <stdio.h>
// #include <stdlib.h>
// #include <dlfcn.h>
// #include "vox.h"
import "C"

import (
    "errors"
    "fmt"
    "unsafe"
    "runtime"
)

func init() {
    runtime.LockOSThread()
}

const (
    // Init flags
    NO_DEBUG_OUTPUT = C.SV_INIT_FLAG_NO_DEBUG_OUTPUT
    USER_AUDIO_CALLBACK = C.SV_INIT_FLAG_USER_AUDIO_CALLBACK
    AUDIO_INT16 = C.SV_INIT_FLAG_AUDIO_INT16
    AUDIO_FLOAT32 = C.SV_INIT_FLAG_AUDIO_FLOAT32
    ONE_THREAD = C.SV_INIT_FLAG_ONE_THREAD

    // Module flags
    FLAG_EXISTS = C.SV_MODULE_FLAG_EXISTS
    FLAG_EFFECT = C.SV_MODULE_FLAG_EFFECT
    INPUTS_OFF = C.SV_MODULE_INPUTS_OFF
    INPUTS_MASK = C.SV_MODULE_INPUTS_MASK
    OUTPUTS_OFF = C.SV_MODULE_OUTPUTS_OFF
    OUTPUTS_MASK = C.SV_MODULE_OUTPUTS_MASK

    // Type flags
    INT16 = C.SV_STYPE_INT16
    INT32 = C.SV_STYPE_INT32
    FLOAT32 = C.SV_STYPE_FLOAT32
    FLOAT64 = C.SV_STYPE_FLOAT64
)

var (
    Version string
    slots = 0
)

func Init(dev string, freq, channels, flags int) error {
    if C.sv_load_dll() != C.int(0) {
        return errors.New("Could not load sunvox library")
    }

    device := C.CString(dev)
    defer C.free(unsafe.Pointer(device))

    ver := int(C.vox_init(device, C.int(freq), C.int(channels), C.int(flags)))
    if ver < 0 {
        return errors.New("Could not initialize sunvox library")
    }
    Version = fmt.Sprintf("%d.%d.%d", (ver>>16)&255, (ver>>8)&255, ver&255)

    return nil
}

func Quit() error {
    if C.vox_deinit() != C.int(0) {
        return errors.New("Problem uninitializing sunvox library")
    }

    if C.sv_load_dll() != C.int(0) {
        return errors.New("Problem unloading sunvox library")
    }

    return nil
}

func SampleType() int {
    return int(C.vox_get_sample_type())
}

type Slot int

func Open(path string) (Slot, error) {
    slot := slots
    if C.vox_open_slot(C.int(slot)) != C.int(0) {
        return -1, errors.New("Could not open new slot")
    }

    name := C.CString(path)
    defer C.free(unsafe.Pointer(name))
    if C.vox_load(C.int(slot), name) != C.int(0) {
        return -1, errors.New(fmt.Sprintf("Could not open song %s", path))
    }

    slots++
    return Slot(slot), nil
}

func (s Slot) Close() error {
    if C.vox_close_slot(C.int(s)) != C.int(0) {
        return errors.New(fmt.Sprintf("Problem closing slot %v", s))
    }
    return nil
}

func (s Slot) SetVolume(vol int) error {
    if C.vox_volume(C.int(s), C.int(vol)) != C.int(0) {
        return errors.New(fmt.Sprintf("Could not change slot %v's volume to %v", s, vol))
    }
    return nil
}

func (s Slot) Play() error {
    if C.vox_play(C.int(s)) != C.int(0) {
        return errors.New(fmt.Sprintf("Could not play slot %v", s))
    }
    return nil
}

func (s Slot) Line() int {
    return int(C.vox_get_current_line(C.int(s)))
}
<commit_msg>Add docs to current API.<commit_after>package vox

// #cgo LDFLAGS: -lstdc++ -ldl -lm
// #define SUNVOX_MAIN
// #include <stdio.h>
// #include <stdlib.h>
// #include <dlfcn.h>
// #include "vox.h"
import "C"

import (
    "errors"
    "fmt"
    "unsafe"
    "runtime"
)

func init() {
    runtime.LockOSThread()
}

const (
    // Init flags
    NO_DEBUG_OUTPUT = C.SV_INIT_FLAG_NO_DEBUG_OUTPUT
    USER_AUDIO_CALLBACK = C.SV_INIT_FLAG_USER_AUDIO_CALLBACK
    AUDIO_INT16 = C.SV_INIT_FLAG_AUDIO_INT16
    AUDIO_FLOAT32 = C.SV_INIT_FLAG_AUDIO_FLOAT32
    ONE_THREAD = C.SV_INIT_FLAG_ONE_THREAD

    // Module flags
    FLAG_EXISTS = C.SV_MODULE_FLAG_EXISTS
    FLAG_EFFECT = C.SV_MODULE_FLAG_EFFECT
    INPUTS_OFF = C.SV_MODULE_INPUTS_OFF
    INPUTS_MASK = C.SV_MODULE_INPUTS_MASK
    OUTPUTS_OFF = C.SV_MODULE_OUTPUTS_OFF
    OUTPUTS_MASK = C.SV_MODULE_OUTPUTS_MASK

    // Type flags
    INT16 = C.SV_STYPE_INT16
    INT32 = C.SV_STYPE_INT32
    FLOAT32 = C.SV_STYPE_FLOAT32
    FLOAT64 = C.SV_STYPE_FLOAT64
)

var (
    Version string
    slots = 0
)

// Init loads the sunvox dll and initializes the library.
func Init(dev string, freq, channels, flags int) error {
    if C.sv_load_dll() != C.int(0) {
        return errors.New("Could not load sunvox library")
    }

    device := C.CString(dev)
    defer C.free(unsafe.Pointer(device))

    ver := int(C.vox_init(device, C.int(freq), C.int(channels), C.int(flags)))
    if ver < 0 {
        return errors.New("Could not initialize sunvox library")
    }
    Version = fmt.Sprintf("%d.%d.%d", (ver>>16)&255, (ver>>8)&255, ver&255)

    return nil
}

// Quit deinitializes the library and unloads the sunvox dll.
func Quit() error {
    if C.vox_deinit() != C.int(0) {
        return errors.New("Problem uninitializing sunvox library")
    }

    if C.sv_load_dll() != C.int(0) {
        return errors.New("Problem unloading sunvox library")
    }

    return nil
}

// SampleType returns the internal sample type of the sunvox engine.
func SampleType() int {
    return int(C.vox_get_sample_type())
}

// Slot is used to load and play sunvox songs.
type Slot int

// Open creates a new slot and laods a sunvox song into it.
func Open(path string) (Slot, error) {
    slot := slots
    if C.vox_open_slot(C.int(slot)) != C.int(0) {
        return -1, errors.New("Could not open new slot")
    }

    name := C.CString(path)
    defer C.free(unsafe.Pointer(name))
    if C.vox_load(C.int(slot), name) != C.int(0) {
        return -1, errors.New(fmt.Sprintf("Could not open song %s", path))
    }

    slots++
    return Slot(slot), nil
}

// Close closes the slot. The slot should no longer be used after calling it.
func (s Slot) Close() error {
    if C.vox_close_slot(C.int(s)) != C.int(0) {
        return errors.New(fmt.Sprintf("Problem closing slot %v", s))
    }
    return nil
}

// SetVolume sets the volume of the slot.
func (s Slot) SetVolume(vol int) error {
    if C.vox_volume(C.int(s), C.int(vol)) != C.int(0) {
        return errors.New(fmt.Sprintf("Could not change slot %v's volume to %v", s, vol))
    }
    return nil
}

// Play starts playback from where ever the song was stopped.
func (s Slot) Play() error {
    if C.vox_play(C.int(s)) != C.int(0) {
        return errors.New(fmt.Sprintf("Could not play slot %v", s))
    }
    return nil
}

// Line returns the current line in the song.
func (s Slot) Line() int {
    return int(C.vox_get_current_line(C.int(s)))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/smtp"
	"os"
)

func usage() {
	fmt.Printf("Usage: %s <host> <address>\n", os.Args[0])
}

func main() {
	args := os.Args[1:]
	if len(args) != 2 {
		usage()
		os.Exit(1)
	}

	host := fmt.Sprintf("%s:25", args[0])
	address := args[1]

	cli, err := smtp.Dial(host)

	if err != nil {
		fmt.Println("Error", err)
		os.Exit(2)
	}

	merr := cli.Mail("foo@mysite.com")
	if merr != nil {
		fmt.Println("Error", merr)
		os.Exit(2)
	}

	rcpterr := cli.Rcpt(address)
	if rcpterr != nil {
		fmt.Printf("Address %s is probably invalid\n", address)
		fmt.Printf("(Server said: %s)\n", rcpterr)
		os.Exit(2)
	} else {
		fmt.Printf("Address %s is valid\n", address)
	}

	reseterr := cli.Reset()
	if reseterr != nil {
		fmt.Println("Error", reseterr)
		os.Exit(2)
	}

	qerr := cli.Quit()
	if qerr != nil {
		fmt.Println("Error", qerr)
		os.Exit(2)
	}

	cli.Close()
}
<commit_msg>refactor deliverable into own function<commit_after>package main

import (
	"fmt"
	"net/smtp"
	"os"
)

func usage() {
	fmt.Printf("Usage: %s <host> <address>\n", os.Args[0])
}

func is_deliverable(host string, address string) (bool, error) {
	deliverable := false

	cli, err := smtp.Dial(host)
	if err != nil {
		return false, err
	}
	defer cli.Close()

	err = cli.Mail("foo@mysite.com")
	if err != nil {
		return false, err
	}

	err = cli.Rcpt(address)
	if err != nil {
		return false, err
	}

	err = cli.Reset()
	if err != nil {
		return deliverable, err
	}

	err = cli.Quit()
	if err != nil {
		return deliverable, err
	}

	return deliverable, nil
}

func main() {
	args := os.Args[1:]
	if len(args) != 2 {
		usage()
		os.Exit(1)
	}

	host := fmt.Sprintf("%s:25", args[0])
	address := args[1]

	deliverable, err := is_deliverable(host, address)
	if err != nil {
		fmt.Println("Error:", err)
	}

	if deliverable {
		fmt.Println(address, "is deliverable")
		os.Exit(0)
	} else {
		fmt.Println(address, "is not deliverable")
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
)

var verbose bool

var ext string = ".was"

func init() {
	flag.BoolVar(&verbose, "v", false, "verbose output")
}

func main() {

	flag.Usage = func() {

		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, `

	Examples:

was filename1 [filename2 filename3 ...]

Move list of files to files with a .was extension, and move them back if they already have a .was extension.

WIP

Make it return non-zero if there were any errors
Let user choose the extension

`)

		flag.PrintDefaults()
	}

	flag.Parse()

	wasFiles := flag.Args()

	if len(wasFiles) < 1 {

		flag.Usage()
		os.Exit(2)

	}

	flag.Parse()

	if verbose {
		fmt.Println("hello world:%v:%s:", verbose, wasFiles)
	}

	for _, file := range wasFiles {

		if verbose {
			fmt.Fprintf(os.Stderr, "handling file:%s:len(file):%d:\n", file, len(file))
		}

                //chop off slash from directories
                if file[len(file) - 1] == "/"[0] {
                  file = file[0:len(file) - 1]
                }

		if file == ext {
			fmt.Fprintf(os.Stderr, "ignoring .was:%v\n")
			continue
		}

		if _, err := os.Stat(file); err != nil {
			fmt.Fprintf(os.Stderr, "skipping:%v\n", err)
			continue
		}

		targetFile := file + ext
		if strings.HasSuffix(file, ext) {

			if verbose {
				fmt.Fprintf(os.Stderr, "doing unwas on:%s\n", targetFile)
			}

			targetFile = file[0 : len(file)-len(ext)]

		}

		if _, err := os.Stat(targetFile); err == nil {

			if verbose {
				fmt.Fprintf(os.Stderr, "target is blocked:%s\n", targetFile)
			}

			if err := os.Remove(targetFile); err != nil {
				fmt.Fprintf(os.Stderr, "could not clear the way for new was file:skipping:%v\n", err)
				continue
			}
		}

		if verbose {
			fmt.Fprintf(os.Stderr, "target is clear:%s\n", file)
		}

		if err := os.Rename(file, targetFile); err != nil {
			fmt.Fprintf(os.Stderr, "failed to was:%v\n", err)
			continue
		}

		if verbose {
			fmt.Fprintf(os.Stderr, "was'd:%s\n", file)
		}

	}

}
<commit_msg>make it prompt before deletion<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"strings"
        "log"
)

var verbose bool

var ext string = ".was"

func init() {
	flag.BoolVar(&verbose, "v", false, "verbose output")
}

//swiped this from a gist:
//https://gist.github.com/albrow/5882501
func askForConfirmation() bool {
        consolereader := bufio.NewReader(os.Stdin)

        response, err := consolereader.ReadString('\n')
	if err != nil {
		log.Fatal(err)
	}
	okayResponses := []string{"y", "Y", "yes", "Yes", "YES"}
	nokayResponses := []string{"n", "N", "no", "No", "NO"}
	if containsString(okayResponses, response[:len(response) - 1]) {
		return true
	} else if containsString(nokayResponses, response[: len(response) - 1]) {
		return false
	} else {
		fmt.Println("Please type yes or no and then press enter:")
		return askForConfirmation()
	}
}

// posString returns the first index of element in slice.
// If slice does not contain element, returns -1.
func posString(slice []string, element string) int {
	for index, elem := range slice {
		if elem == element {
			return index
		}
	}
	return -1
}

// containsString returns true iff slice contains element
func containsString(slice []string, element string) bool {
	return !(posString(slice, element) == -1)
}

func main() {

	flag.Usage = func() {

		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, `

        Description:

        Stupid simple but useful tool to move a file and move it back later.

	Examples:

was filename1 [filename2 filename3 ...]

Move list of files to files with a .was extension, and move them back if they already have a .was extension.

WIP

Make it return non-zero if there were any errors
Let user choose the extension.
Read file list from STDIN

`)

		flag.PrintDefaults()
	}

	flag.Parse()

	wasFiles := flag.Args()

	if len(wasFiles) < 1 {

		flag.Usage()
		os.Exit(2)

	}

	flag.Parse()

	if verbose {
		fmt.Println("hello world:%v:%s:", verbose, wasFiles)
	}

	for _, file := range wasFiles {

		if verbose {
			fmt.Fprintf(os.Stderr, "handling file:%s:len(file):%d:\n", file, len(file))
		}

		//chop off slash from directories
		if file[len(file)-1] == '/' {
			file = file[0 : len(file)-1]
		}

		if file == ext {
			fmt.Fprintf(os.Stderr, "ignoring .was:%v\n")
			continue
		}

		if _, err := os.Stat(file); err != nil {
			fmt.Fprintf(os.Stderr, "skipping:%v\n", err)
			continue
		}

		targetFile := file + ext
		if strings.HasSuffix(file, ext) {

			if verbose {
				fmt.Fprintf(os.Stderr, "doing unwas on:%s\n", targetFile)
			}

			targetFile = file[0 : len(file)-len(ext)]

		}

		if _, err := os.Stat(targetFile); err == nil {

			if verbose {
				fmt.Fprintf(os.Stderr, "target is blocked:%s\n", targetFile)
			}

		        fmt.Printf("There's a file in the way:%s:\n", targetFile)
		        fmt.Printf("Delete %s? Please type yes or no and then press enter:\n", targetFile)
                        if askForConfirmation() {
			  if err := os.RemoveAll(targetFile); err != nil {
				fmt.Fprintf(os.Stderr, "could not clear the way for new was file:skipping:%v\n", err)
				continue
			  }
                        } else {
				fmt.Fprintf(os.Stderr, "user chose to not delete target:skipping:%s\n", targetFile)
				continue
                        }
		}

		if verbose {
			fmt.Fprintf(os.Stderr, "target is clear:%s\n", file)
		}

		if err := os.Rename(file, targetFile); err != nil {
			fmt.Fprintf(os.Stderr, "failed to was:%v\n", err)
			continue
		}

		if verbose {
			fmt.Fprintf(os.Stderr, "was'd:%s\n", file)
		}

	}

}
<|endoftext|>


<commit_before>package main

import (
	"github.com/ant0ine/go-json-rest/rest"
	"net/http"
)

func newHandler() rest.ResourceHandler {
	handler := rest.ResourceHandler{}
	handler.SetRoutes(
		&rest.Route{"GET", "/verify/", getEmailVerify},
	)
	return handler
}

func main() {
	handler := newHandler()
	http.ListenAndServe(":8080", &handler)
}

type Message struct {
	Email   string `json:"email"`
	IsValid bool   `json:"isValid"`
}

func getEmailVerify(w rest.ResponseWriter, req *rest.Request) {
	email := req.URL.Query().Get("email")
	valid, _ := VerifyEmail(email)

	w.WriteJson(&Message{
		Email:   email,
		IsValid: valid,
	})
}
<commit_msg>web main get port from environment var<commit_after>package main

import (
	"github.com/ant0ine/go-json-rest/rest"
	"log"
	"net/http"
	"os"
)

func newHandler() rest.ResourceHandler {
	handler := rest.ResourceHandler{}
	handler.SetRoutes(
		&rest.Route{"GET", "/verify/", getEmailVerify},
	)
	return handler
}

func main() {
	PORT := os.Getenv("PORT")

	handler := newHandler()

	log.Println("Listening on port " + PORT)
	err := http.ListenAndServe(":"+PORT, &handler)
	if err != nil {
		log.Panic(err)
	}
}

type Message struct {
	Email   string `json:"email"`
	IsValid bool   `json:"isValid"`
}

func getEmailVerify(w rest.ResponseWriter, req *rest.Request) {
	email := req.URL.Query().Get("email")
	valid, _ := VerifyEmail(email)

	w.WriteJson(&Message{
		Email:   email,
		IsValid: valid,
	})
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Che Wei, Lin
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tinynet

import (
	"io"
	"io/ioutil"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/client"
	"golang.org/x/net/context"
)

func ensureDocker(imageRef string) (containerID string, sandboxKey string, err error) {
	ctx := context.Background()
	cli, err := client.NewEnvClient()
	if err != nil {
		return "", "", err
	}

	// docker pull busybox
	readCloser, err := cli.ImagePull(ctx, imageRef, types.ImagePullOptions{})
	if err != nil {
		return "", "", err
	}
	// because readCloser need to be handle so that image can be download.
	// we don't need output so send this to /dev/null
	io.Copy(ioutil.Discard, readCloser)

	// docker run --net=none -d busybox sleep 3600
	resp, err := cli.ContainerCreate(ctx, &container.Config{
		Image:           imageRef,
		Cmd:             []string{"sleep", "3600"},
		NetworkDisabled: false,
	}, nil, nil, "")
	if err != nil {
		return "", "", err
	}

	if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
		return "", "", err
	}

	var cInfo types.ContainerJSON
	// docker inspect bb | grep -E 'SandboxKey|Id'
	cInfo, err = cli.ContainerInspect(ctx, resp.ID)
	if err != nil {
		return "", "", err
	}
	return resp.ID, cInfo.NetworkSettings.SandboxKey, err
}
<commit_msg>set network mode to none<commit_after>// Copyright (c) 2017 Che Wei, Lin
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tinynet

import (
	"io"
	"io/ioutil"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/client"
	"golang.org/x/net/context"
)

func ensureDocker(imageRef string) (containerID string, sandboxKey string, err error) {
	ctx := context.Background()
	cli, err := client.NewEnvClient()
	if err != nil {
		return "", "", err
	}

	// docker pull busybox
	readCloser, err := cli.ImagePull(ctx, imageRef, types.ImagePullOptions{})
	if err != nil {
		return "", "", err
	}
	// because readCloser need to be handle so that image can be download.
	// we don't need output so send this to /dev/null
	io.Copy(ioutil.Discard, readCloser)

	// docker run --net=none -d busybox sleep 3600
	resp, err := cli.ContainerCreate(ctx, &container.Config{
		Image: imageRef,
		Cmd:   []string{"while", "true;", "do", "sleep", "3600;", "done;"},
		// Cmd:   []string{"sleep", "3600"},
	}, &container.HostConfig{
		NetworkMode: "none",
	}, nil, "")
	if err != nil {
		return "", "", err
	}

	if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
		return "", "", err
	}

	var cInfo types.ContainerJSON
	// docker inspect bb | grep -E 'SandboxKey|Id'
	cInfo, err = cli.ContainerInspect(ctx, resp.ID)
	if err != nil {
		return "", "", err
	}
	return resp.ID, cInfo.NetworkSettings.SandboxKey, err
}
<|endoftext|>


<commit_before>package cortexbot

import (
	"regexp"
	"strings"

	valid "github.com/asaskevich/govalidator"
)

const (
	// Modified version of govalidator.DNSName that does not allow domain without tld (like localhost)
	DNSName string = `^([a-zA-Z0-9_]{1}[a-zA-Z0-9_-]{0,62}){1}(\.[a-zA-Z0-9_]{1}[a-zA-Z0-9_-]{0,62})+[\._]?$`
	Hash    string = `^[a-fA-F0-9]{32,}$`
)

var (
	rxDNS  = regexp.MustCompile(DNSName)
	rxHash = regexp.MustCompile(Hash)
)

// IsHash checks if a given string is a hash
// BUG(ilyaglow): not supported hashes are hashes shorter 32 letters and ssdeep
func IsHash(str string) bool {
	return rxHash.MatchString(str)
}

// IsDNSName is a modified version of function IsDNSName from https://github.com/asaskevich/govalidator/blob/master/patterns.go
func IsDNSName(str string) bool {
	if str == "" || len(strings.Replace(str, ".", "", -1)) > 255 {
		return false
	}
	return !valid.IsIP(str) && rxDNS.MatchString(str)
}
<commit_msg>Unexport package variables<commit_after>package cortexbot

import (
	"regexp"
	"strings"

	valid "github.com/asaskevich/govalidator"
)

const (
	// dnsName is a modified version of govalidator.DNSName that does not allow domain without tld (like localhost)
	dnsName string = `^([a-zA-Z0-9_]{1}[a-zA-Z0-9_-]{0,62}){1}(\.[a-zA-Z0-9_]{1}[a-zA-Z0-9_-]{0,62})+[\._]?$`
	// hash is any string contains hex letters and longer that 32 symbols
	hash string = `^[a-fA-F0-9]{32,}$`
)

var (
	rxDNS  = regexp.MustCompile(dnsName)
	rxHash = regexp.MustCompile(hash)
)

// IsHash checks if a given string is a hash
// BUG(ilyaglow): not supported hashes are hashes shorter 32 letters and ssdeep
func IsHash(str string) bool {
	return rxHash.MatchString(str)
}

// IsDNSName is a modified version of function IsDNSName from https://github.com/asaskevich/govalidator/blob/master/patterns.go
func IsDNSName(str string) bool {
	if str == "" || len(strings.Replace(str, ".", "", -1)) > 255 {
		return false
	}
	return !valid.IsIP(str) && rxDNS.MatchString(str)
}
<|endoftext|>


<commit_before>package peering

import (
	"context"
	"errors"
	"fmt"
	"time"

	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/go-memdb"

	"github.com/hashicorp/consul/agent/cache"
	"github.com/hashicorp/consul/agent/structs"
	"github.com/hashicorp/consul/agent/submatview"
	"github.com/hashicorp/consul/lib/retry"
	"github.com/hashicorp/consul/proto/pbservice"
)

type MaterializedViewStore interface {
	Get(ctx context.Context, req submatview.Request) (submatview.Result, error)
	Notify(ctx context.Context, req submatview.Request, cID string, ch chan<- cache.UpdateEvent) error
}

type SubscriptionBackend interface {
	Subscriber
	Store() Store
}

// subscriptionManager handlers requests to subscribe to events from an events publisher.
type subscriptionManager struct {
	logger    hclog.Logger
	viewStore MaterializedViewStore
	backend   SubscriptionBackend

	// watchedServices is a map of exported services to a cancel function for their subscription notifier.
	watchedServices map[structs.ServiceName]context.CancelFunc
}

// TODO(peering): Maybe centralize so that there is a single manager per datacenter, rather than per peering.
func newSubscriptionManager(ctx context.Context, logger hclog.Logger, backend SubscriptionBackend) *subscriptionManager {
	logger = logger.Named("subscriptions")
	store := submatview.NewStore(logger.Named("viewstore"))
	go store.Run(ctx)

	return &subscriptionManager{
		logger:          logger,
		viewStore:       store,
		backend:         backend,
		watchedServices: make(map[structs.ServiceName]context.CancelFunc),
	}
}

// subscribe returns a channel that will contain updates to exported service instances for a given peer.
func (m *subscriptionManager) subscribe(ctx context.Context, peerID string) <-chan cache.UpdateEvent {
	updateCh := make(chan cache.UpdateEvent, 1)
	go m.syncSubscriptions(ctx, peerID, updateCh)

	return updateCh
}

func (m *subscriptionManager) syncSubscriptions(ctx context.Context, peerID string, updateCh chan<- cache.UpdateEvent) {
	waiter := &retry.Waiter{
		MinFailures: 1,
		Factor:      500 * time.Millisecond,
		MaxWait:     60 * time.Second,
		Jitter:      retry.NewJitter(100),
	}

	for {
		if err := m.syncSubscriptionsAndBlock(ctx, peerID, updateCh); err != nil {
			m.logger.Error("failed to sync subscriptions", "error", err)
		}

		if err := waiter.Wait(ctx); err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
			m.logger.Error("failed to wait before re-trying sync", "error", err)
		}

		select {
		case <-ctx.Done():
			return
		default:
		}
	}
}

// syncSubscriptionsAndBlock ensures that the subscriptions to the subscription backend
// match the list of services exported to the peer.
func (m *subscriptionManager) syncSubscriptionsAndBlock(ctx context.Context, peerID string, updateCh chan<- cache.UpdateEvent) error {
	store := m.backend.Store()

	ws := memdb.NewWatchSet()
	ws.Add(store.AbandonCh())
	ws.Add(ctx.Done())

	// Get exported services for peer id
	_, services, err := store.ExportedServicesForPeer(ws, peerID)
	if err != nil {
		return fmt.Errorf("failed to watch exported services for peer %q: %w", peerID, err)
	}

	// seen contains the set of exported service names and is used to reconcile the list of watched services.
	seen := make(map[structs.ServiceName]struct{})

	// Ensure there is a subscription for each service exported to the peer.
	for _, svc := range services {
		seen[svc] = struct{}{}

		if _, ok := m.watchedServices[svc]; ok {
			// Exported service is already being watched, nothing to do.
			continue
		}

		notifyCtx, cancel := context.WithCancel(ctx)
		m.watchedServices[svc] = cancel

		if err := m.Notify(notifyCtx, svc, updateCh); err != nil {
			m.logger.Error("failed to subscribe to service", "service", svc.String())
			continue
		}
	}

	// For every subscription without an exported service, call the associated cancel fn.
	for svc, cancel := range m.watchedServices {
		if _, ok := seen[svc]; !ok {
			cancel()

			// Send an empty event to the stream handler to trigger sending a DELETE message.
			// Cancelling the subscription context above is necessary, but does not yield a useful signal on its own.
			updateCh <- cache.UpdateEvent{
				CorrelationID: subExportedService + svc.String(),
				Result:        &pbservice.IndexedCheckServiceNodes{},
			}
		}
	}

	// Block for any changes to the state store.
	ws.WatchCh(ctx)
	return nil
}

const (
	subExportedService = "exported-service:"
)

// Notify the given channel when there are updates to the requested service.
func (m *subscriptionManager) Notify(ctx context.Context, svc structs.ServiceName, updateCh chan<- cache.UpdateEvent) error {
	sr := newExportedServiceRequest(m.logger, svc, m.backend)
	return m.viewStore.Notify(ctx, sr, subExportedService+svc.String(), updateCh)
}
<commit_msg>Actually block when syncing subscriptions (#13066)<commit_after>package peering

import (
	"context"
	"errors"
	"fmt"
	"time"

	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/go-memdb"

	"github.com/hashicorp/consul/agent/cache"
	"github.com/hashicorp/consul/agent/structs"
	"github.com/hashicorp/consul/agent/submatview"
	"github.com/hashicorp/consul/lib/retry"
	"github.com/hashicorp/consul/proto/pbservice"
)

type MaterializedViewStore interface {
	Get(ctx context.Context, req submatview.Request) (submatview.Result, error)
	Notify(ctx context.Context, req submatview.Request, cID string, ch chan<- cache.UpdateEvent) error
}

type SubscriptionBackend interface {
	Subscriber
	Store() Store
}

// subscriptionManager handlers requests to subscribe to events from an events publisher.
type subscriptionManager struct {
	logger    hclog.Logger
	viewStore MaterializedViewStore
	backend   SubscriptionBackend

	// watchedServices is a map of exported services to a cancel function for their subscription notifier.
	watchedServices map[structs.ServiceName]context.CancelFunc
}

// TODO(peering): Maybe centralize so that there is a single manager per datacenter, rather than per peering.
func newSubscriptionManager(ctx context.Context, logger hclog.Logger, backend SubscriptionBackend) *subscriptionManager {
	logger = logger.Named("subscriptions")
	store := submatview.NewStore(logger.Named("viewstore"))
	go store.Run(ctx)

	return &subscriptionManager{
		logger:          logger,
		viewStore:       store,
		backend:         backend,
		watchedServices: make(map[structs.ServiceName]context.CancelFunc),
	}
}

// subscribe returns a channel that will contain updates to exported service instances for a given peer.
func (m *subscriptionManager) subscribe(ctx context.Context, peerID string) <-chan cache.UpdateEvent {
	updateCh := make(chan cache.UpdateEvent, 1)
	go m.syncSubscriptions(ctx, peerID, updateCh)

	return updateCh
}

func (m *subscriptionManager) syncSubscriptions(ctx context.Context, peerID string, updateCh chan<- cache.UpdateEvent) {
	waiter := &retry.Waiter{
		MinFailures: 1,
		Factor:      500 * time.Millisecond,
		MaxWait:     60 * time.Second,
		Jitter:      retry.NewJitter(100),
	}

	for {
		if err := m.syncSubscriptionsAndBlock(ctx, peerID, updateCh); err != nil {
			m.logger.Error("failed to sync subscriptions", "error", err)
		}

		if err := waiter.Wait(ctx); err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
			m.logger.Error("failed to wait before re-trying sync", "error", err)
		}

		select {
		case <-ctx.Done():
			return
		default:
		}
	}
}

// syncSubscriptionsAndBlock ensures that the subscriptions to the subscription backend
// match the list of services exported to the peer.
func (m *subscriptionManager) syncSubscriptionsAndBlock(ctx context.Context, peerID string, updateCh chan<- cache.UpdateEvent) error {
	store := m.backend.Store()

	ws := memdb.NewWatchSet()
	ws.Add(store.AbandonCh())
	ws.Add(ctx.Done())

	// Get exported services for peer id
	_, services, err := store.ExportedServicesForPeer(ws, peerID)
	if err != nil {
		return fmt.Errorf("failed to watch exported services for peer %q: %w", peerID, err)
	}

	// seen contains the set of exported service names and is used to reconcile the list of watched services.
	seen := make(map[structs.ServiceName]struct{})

	// Ensure there is a subscription for each service exported to the peer.
	for _, svc := range services {
		seen[svc] = struct{}{}

		if _, ok := m.watchedServices[svc]; ok {
			// Exported service is already being watched, nothing to do.
			continue
		}

		notifyCtx, cancel := context.WithCancel(ctx)
		m.watchedServices[svc] = cancel

		if err := m.Notify(notifyCtx, svc, updateCh); err != nil {
			m.logger.Error("failed to subscribe to service", "service", svc.String())
			continue
		}
	}

	// For every subscription without an exported service, call the associated cancel fn.
	for svc, cancel := range m.watchedServices {
		if _, ok := seen[svc]; !ok {
			cancel()

			// Send an empty event to the stream handler to trigger sending a DELETE message.
			// Cancelling the subscription context above is necessary, but does not yield a useful signal on its own.
			updateCh <- cache.UpdateEvent{
				CorrelationID: subExportedService + svc.String(),
				Result:        &pbservice.IndexedCheckServiceNodes{},
			}
		}
	}

	// Block for any changes to the state store.
	ws.WatchCtx(ctx)
	return nil
}

const (
	subExportedService = "exported-service:"
)

// Notify the given channel when there are updates to the requested service.
func (m *subscriptionManager) Notify(ctx context.Context, svc structs.ServiceName, updateCh chan<- cache.UpdateEvent) error {
	sr := newExportedServiceRequest(m.logger, svc, m.backend)
	return m.viewStore.Notify(ctx, sr, subExportedService+svc.String(), updateCh)
}
<|endoftext|>


<commit_before>package cache

import (
	"fmt"

	"log"
	"math"
	"os"
	"runtime"
	"strconv"
	"sync"
	"time"

	"github.com/go-redis/redis"
	"github.com/syou6162/go-active-learning/lib/example"
	"github.com/syou6162/go-active-learning/lib/feature"
	"github.com/syou6162/go-active-learning/lib/fetcher"
	"github.com/syou6162/go-active-learning/lib/util"
)

type Cache struct {
	Client *redis.Client
}

var redisPrefix = "url"

func NewCache() (*Cache, error) {
	host := util.GetEnv("REDIS_HOST", "localhost")
	client := redis.NewClient(&redis.Options{
		Addr:        fmt.Sprintf("%s:6379", host),
		Password:    "", // no password set
		DB:          0,  // use default DB
		PoolSize:    100,
		MaxRetries:  4,
		PoolTimeout: time.Duration(10) * time.Second,
		IdleTimeout: time.Duration(60) * time.Second,
	})

	_, err := client.Ping().Result()
	if err != nil {
		return nil, err
	}
	return &Cache{Client: client}, nil
}

func (c *Cache) Close() error {
	return c.Client.Close()
}

func (c *Cache) attachMetadata(examples example.Examples) error {
	for _, e := range examples {
		key := redisPrefix + ":" + e.Url
		vals, err := c.Client.HMGet(key,
			"Fv",            // 0
			"FinalUrl",      // 1
			"Title",         // 2
			"Description",   // 3
			"OgDescription", // 4
			"Body",          // 5
			"Score",         // 6
			"IsNew",         // 7
			"StatusCode",    // 8
		).Result()
		if err != nil {
			return err
		}

		// Fv
		if result, ok := vals[0].(feature.FeatureVector); ok {
			e.Fv = result
		}
		// FinalUrl
		if result, ok := vals[1].(string); ok {
			e.FinalUrl = result
		}
		// Title
		if result, ok := vals[2].(string); ok {
			e.Title = result
		}
		// Description
		if result, ok := vals[3].(string); ok {
			e.Description = result
		}
		// OgDescription
		if result, ok := vals[4].(string); ok {
			e.OgDescription = result
		}
		// Body
		if result, ok := vals[5].(string); ok {
			e.Body = result
		}
		// Score
		if result, ok := vals[6].(float64); ok {
			e.Score = result
		}
		// IsNew
		if result, ok := vals[7].(bool); ok {
			e.IsNew = result
		}
		// StatusCode
		if result, ok := vals[8].(int); ok {
			e.StatusCode = result
		}
	}
	return nil
}

func (c *Cache) attachLightMetadata(examples example.Examples) error {
	for _, e := range examples {
		key := redisPrefix + ":" + e.Url
		vals, err := c.Client.HMGet(key,
			"FinalUrl",      // 0
			"Title",         // 1
			"Description",   // 2
			"OgDescription", // 3
			"Body",          // 4
			"Score",         // 5
			"StatusCode",    // 6
		).Result()
		if err != nil {
			return err
		}

		// FinalUrl
		if result, ok := vals[0].(string); ok {
			e.FinalUrl = result
		}
		// Title
		if result, ok := vals[1].(string); ok {
			e.Title = result
		}
		// Description
		if result, ok := vals[2].(string); ok {
			e.Description = result
		}
		// OgDescription
		if result, ok := vals[3].(string); ok {
			e.OgDescription = result
		}
		// Body
		if result, ok := vals[4].(string); ok {
			e.Body = result
		}
		// Score
		if result, ok := vals[4].(float64); ok {
			e.Score = result
		}
		// StatusCode
		if result, ok := vals[5].(int); ok {
			e.StatusCode = result
		}
	}
	return nil
}

func fetchMetaData(e *example.Example) {
	article := fetcher.GetArticle(e.Url)
	e.Title = article.Title
	e.FinalUrl = article.Url
	e.Description = article.Description
	e.OgDescription = article.OgDescription
	e.Body = article.Body
	e.StatusCode = article.StatusCode
	e.Fv = util.RemoveDuplicate(example.ExtractFeatures(*e))
}

func (c *Cache) SetExample(example example.Example) error {
	key := redisPrefix + ":" + example.Url

	vals := make(map[string]interface{})
	vals["Label"] = &example.Label
	vals["Fv"] = &example.Fv
	vals["Url"] = example.Url
	vals["FinalUrl"] = example.FinalUrl
	vals["Title"] = example.Title
	vals["Description"] = example.Description
	vals["OgDescription"] = example.OgDescription
	vals["Body"] = example.Body
	vals["Score"] = example.Score
	vals["IsNew"] = example.IsNew
	vals["StatusCode"] = example.StatusCode

	if err := c.Client.HMSet(key, vals).Err(); err != nil {
		return err
	}
	if err := c.Client.Expire(key, time.Hour*240).Err(); err != nil {
		return err
	}
	return nil
}

func (cache *Cache) AttachMetadata(examples example.Examples, fetchNewExamples bool, useLightMetadata bool) {
	batchSize := 100
	examplesList := make([]example.Examples, 0)
	n := len(examples)

	for i := 0; i < n; i += batchSize {
		max := int(math.Min(float64(i+batchSize), float64(n)))
		examplesList = append(examplesList, examples[i:max])
	}
	for _, l := range examplesList {
		if useLightMetadata {
			if err := cache.attachLightMetadata(l); err != nil {
				log.Println(err.Error())
			}
		} else {
			if err := cache.attachMetadata(l); err != nil {
				log.Println(err.Error())
			}
		}
		if !fetchNewExamples {
			continue
		}
		examplesWithEmptyMetaData := example.Examples{}
		for _, e := range l {
			if e.StatusCode != 200 {
				examplesWithEmptyMetaData = append(examplesWithEmptyMetaData, e)
			}
		}
		wg := &sync.WaitGroup{}
		cpus := runtime.NumCPU()
		runtime.GOMAXPROCS(cpus)
		sem := make(chan struct{}, batchSize)
		for idx, e := range examplesWithEmptyMetaData {
			wg.Add(1)
			sem <- struct{}{}
			go func(e *example.Example, idx int) {
				defer wg.Done()
				fmt.Fprintln(os.Stderr, "Fetching("+strconv.Itoa(idx)+"): "+e.Url)
				fetchMetaData(e)
				if err := cache.SetExample(*e); err != nil {
					log.Println(err.Error())
				}
				<-sem
			}(e, idx)
		}
		wg.Wait()
	}
}

var listPrefix = "list:"

func (c *Cache) AddExamplesToList(listName string, examples example.Examples) error {
	if err := c.Client.Del(listPrefix + listName).Err(); err != nil {
		return err
	}

	result := make([]redis.Z, 0)
	for _, e := range examples {
		url := e.Url
		if e.FinalUrl != "" {
			url = e.FinalUrl
		}
		result = append(result, redis.Z{Score: e.Score, Member: url})
	}
	// ToDo: take care the case when result is empty
	err := c.Client.ZAdd(listPrefix+listName, result...).Err()
	if err != nil {
		return err
	}
	return nil
}

func (c *Cache) GetUrlsFromList(listName string, from int64, to int64) ([]string, error) {
	sliceCmd := c.Client.ZRevRange(listPrefix+listName, from, to)
	if sliceCmd.Err() != nil {
		return nil, sliceCmd.Err()
	}
	return sliceCmd.Val(), nil
}
<commit_msg>数値として解釈させる<commit_after>package cache

import (
	"fmt"

	"log"
	"math"
	"os"
	"runtime"
	"strconv"
	"sync"
	"time"

	"github.com/go-redis/redis"
	"github.com/syou6162/go-active-learning/lib/example"
	"github.com/syou6162/go-active-learning/lib/feature"
	"github.com/syou6162/go-active-learning/lib/fetcher"
	"github.com/syou6162/go-active-learning/lib/util"
)

type Cache struct {
	Client *redis.Client
}

var redisPrefix = "url"

func NewCache() (*Cache, error) {
	host := util.GetEnv("REDIS_HOST", "localhost")
	client := redis.NewClient(&redis.Options{
		Addr:        fmt.Sprintf("%s:6379", host),
		Password:    "", // no password set
		DB:          0,  // use default DB
		PoolSize:    100,
		MaxRetries:  4,
		PoolTimeout: time.Duration(10) * time.Second,
		IdleTimeout: time.Duration(60) * time.Second,
	})

	_, err := client.Ping().Result()
	if err != nil {
		return nil, err
	}
	return &Cache{Client: client}, nil
}

func (c *Cache) Close() error {
	return c.Client.Close()
}

func (c *Cache) attachMetadata(examples example.Examples) error {
	for _, e := range examples {
		key := redisPrefix + ":" + e.Url
		vals, err := c.Client.HMGet(key,
			"Fv",            // 0
			"FinalUrl",      // 1
			"Title",         // 2
			"Description",   // 3
			"OgDescription", // 4
			"Body",          // 5
			"Score",         // 6
			"IsNew",         // 7
			"StatusCode",    // 8
		).Result()
		if err != nil {
			return err
		}

		// Fv
		if result, ok := vals[0].(feature.FeatureVector); ok {
			e.Fv = result
		}
		// FinalUrl
		if result, ok := vals[1].(string); ok {
			e.FinalUrl = result
		}
		// Title
		if result, ok := vals[2].(string); ok {
			e.Title = result
		}
		// Description
		if result, ok := vals[3].(string); ok {
			e.Description = result
		}
		// OgDescription
		if result, ok := vals[4].(string); ok {
			e.OgDescription = result
		}
		// Body
		if result, ok := vals[5].(string); ok {
			e.Body = result
		}
		// Score
		if result, ok := vals[6].(string); ok {
			if score, err := strconv.ParseFloat(result, 64); err == nil {
				e.Score = score
			}
		}
		// IsNew
		if result, ok := vals[7].(string); ok {
			if isNew, err := strconv.ParseBool(result); err == nil {
				e.IsNew = isNew
			}
		}
		// StatusCode
		if result, ok := vals[8].(string); ok {
			if statusCode, err := strconv.Atoi(result); err == nil {
				e.StatusCode = statusCode
			}
		}
	}
	return nil
}

func (c *Cache) attachLightMetadata(examples example.Examples) error {
	for _, e := range examples {
		key := redisPrefix + ":" + e.Url
		vals, err := c.Client.HMGet(key,
			"FinalUrl",      // 0
			"Title",         // 1
			"Description",   // 2
			"OgDescription", // 3
			"Body",          // 4
			"Score",         // 5
			"StatusCode",    // 6
		).Result()
		if err != nil {
			return err
		}

		// FinalUrl
		if result, ok := vals[0].(string); ok {
			e.FinalUrl = result
		}
		// Title
		if result, ok := vals[1].(string); ok {
			e.Title = result
		}
		// Description
		if result, ok := vals[2].(string); ok {
			e.Description = result
		}
		// OgDescription
		if result, ok := vals[3].(string); ok {
			e.OgDescription = result
		}
		// Body
		if result, ok := vals[4].(string); ok {
			e.Body = result
		}
		// Score
		if result, ok := vals[5].(string); ok {
			if score, err := strconv.ParseFloat(result, 64); err == nil {
				e.Score = score
			}
		}
		// StatusCode
		if result, ok := vals[6].(string); ok {
			if statusCode, err := strconv.Atoi(result); err == nil {
				e.StatusCode = statusCode
			}
		}
	}
	return nil
}

func fetchMetaData(e *example.Example) {
	article := fetcher.GetArticle(e.Url)
	e.Title = article.Title
	e.FinalUrl = article.Url
	e.Description = article.Description
	e.OgDescription = article.OgDescription
	e.Body = article.Body
	e.StatusCode = article.StatusCode
	e.Fv = util.RemoveDuplicate(example.ExtractFeatures(*e))
}

func (c *Cache) SetExample(example example.Example) error {
	key := redisPrefix + ":" + example.Url

	vals := make(map[string]interface{})
	vals["Label"] = &example.Label
	vals["Fv"] = &example.Fv
	vals["Url"] = example.Url
	vals["FinalUrl"] = example.FinalUrl
	vals["Title"] = example.Title
	vals["Description"] = example.Description
	vals["OgDescription"] = example.OgDescription
	vals["Body"] = example.Body
	vals["Score"] = example.Score
	vals["IsNew"] = example.IsNew
	vals["StatusCode"] = example.StatusCode

	if err := c.Client.HMSet(key, vals).Err(); err != nil {
		return err
	}
	if err := c.Client.Expire(key, time.Hour*240).Err(); err != nil {
		return err
	}
	return nil
}

func (cache *Cache) AttachMetadata(examples example.Examples, fetchNewExamples bool, useLightMetadata bool) {
	batchSize := 100
	examplesList := make([]example.Examples, 0)
	n := len(examples)

	for i := 0; i < n; i += batchSize {
		max := int(math.Min(float64(i+batchSize), float64(n)))
		examplesList = append(examplesList, examples[i:max])
	}
	for _, l := range examplesList {
		if useLightMetadata {
			if err := cache.attachLightMetadata(l); err != nil {
				log.Println(err.Error())
			}
		} else {
			if err := cache.attachMetadata(l); err != nil {
				log.Println(err.Error())
			}
		}
		if !fetchNewExamples {
			continue
		}
		examplesWithEmptyMetaData := example.Examples{}
		for _, e := range l {
			if e.StatusCode != 200 {
				examplesWithEmptyMetaData = append(examplesWithEmptyMetaData, e)
			}
		}
		wg := &sync.WaitGroup{}
		cpus := runtime.NumCPU()
		runtime.GOMAXPROCS(cpus)
		sem := make(chan struct{}, batchSize)
		for idx, e := range examplesWithEmptyMetaData {
			wg.Add(1)
			sem <- struct{}{}
			go func(e *example.Example, idx int) {
				defer wg.Done()
				fmt.Fprintln(os.Stderr, "Fetching("+strconv.Itoa(idx)+"): "+e.Url)
				fetchMetaData(e)
				if err := cache.SetExample(*e); err != nil {
					log.Println(err.Error())
				}
				<-sem
			}(e, idx)
		}
		wg.Wait()
	}
}

var listPrefix = "list:"

func (c *Cache) AddExamplesToList(listName string, examples example.Examples) error {
	if err := c.Client.Del(listPrefix + listName).Err(); err != nil {
		return err
	}

	result := make([]redis.Z, 0)
	for _, e := range examples {
		url := e.Url
		if e.FinalUrl != "" {
			url = e.FinalUrl
		}
		result = append(result, redis.Z{Score: e.Score, Member: url})
	}
	// ToDo: take care the case when result is empty
	err := c.Client.ZAdd(listPrefix+listName, result...).Err()
	if err != nil {
		return err
	}
	return nil
}

func (c *Cache) GetUrlsFromList(listName string, from int64, to int64) ([]string, error) {
	sliceCmd := c.Client.ZRevRange(listPrefix+listName, from, to)
	if sliceCmd.Err() != nil {
		return nil, sliceCmd.Err()
	}
	return sliceCmd.Val(), nil
}
<|endoftext|>


<commit_before>package framework

import (
	"encoding/json"
	"log"
	"math/rand"
	"os"
	"strconv"
	"sync"

	"github.com/go-distributed/meritop"
)

/*
The dummy task is designed for regresion test of meritop framework.
This works with tree topology.
The main idea behind the regression test is following:
There will be two kinds of dummyTasks: master and slaves. We will have one master
sits at the top with taskID = 0, and then rest 6 (2^n - 2) tasks forms a tree under
the master. There will be 10 epochs, from 1 to 10, at each epoch, we send out a
vector with all values equal to epochID, and each slave is supposedly return a vector
with all values equals epochID*taskID, the values are reduced back to master, and
master will print out the epochID and aggregated vector. After all 10 epoch, it kills
job.
*/

const (
	NumOfIterations uint64 = uint64(10)
)

// dummyData is used to carry parameter and gradient;
type dummyData struct {
	Value int32
}

// dummyMaster is prototype of parameter server, for now it does not
// carry out optimization yet. But it should be easy to add support when
// this full tests out.
// Note: in theory, since there should be no parent of this, so we should
// add error checing in the right places. We will skip these test for now.
type dummyMaster struct {
	dataChan      chan int32
	finishChan    chan struct{}
	NodeProducer  chan bool
	framework     meritop.Framework
	epoch, taskID uint64
	logger        *log.Logger
	config        map[string]string

	param, gradient *dummyData
	fromChildren    map[uint64]*dummyData
}

// This is useful to bring the task up to speed from scratch or if it recovers.
func (t *dummyMaster) Init(taskID uint64, framework meritop.Framework) {
	t.taskID = taskID
	t.framework = framework
	t.logger = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile)
	// t.logger = log.New(ioutil.Discard, "", log.Ldate|log.Ltime|log.Lshortfile)
}

// Task need to finish up for exit, last chance to save work?
func (t *dummyMaster) Exit() {}

// Ideally, we should also have the following:
func (t *dummyMaster) ParentMetaReady(parentID uint64, meta string) {}
func (t *dummyMaster) ChildMetaReady(childID uint64, meta string) {
	t.logger.Printf("master ChildMetaReady, task: %d, epoch: %d, child: %d\n", t.taskID, t.epoch, childID)
	// Get data from child. When all the data is back, starts the next epoch.
	t.framework.DataRequest(childID, meta)
}

// This give the task an opportunity to cleanup and regroup.
func (t *dummyMaster) SetEpoch(epoch uint64) {
	t.logger.Printf("master SetEpoch, task: %d, epoch: %d\n", t.taskID, epoch)
	if t.testablyFail("SetEpoch", strconv.FormatUint(epoch, 10)) {
		return
	}

	t.param = &dummyData{}
	t.gradient = &dummyData{}

	t.epoch = epoch
	t.param.Value = int32(t.epoch)

	// Make sure we have a clean slate.
	t.fromChildren = make(map[uint64]*dummyData)
	t.framework.FlagMetaToChild("ParamReady")
}

// These are payload rpc for application purpose.
func (t *dummyMaster) ServeAsParent(fromID uint64, req string) []byte {
	b, err := json.Marshal(t.param)
	if err != nil {
		t.logger.Fatalf("Master can't encode parameter: %v, error: %v\n", t.param, err)
	}
	return b
}

func (t *dummyMaster) ServeAsChild(fromID uint64, req string) []byte {
	return nil
}

func (t *dummyMaster) ParentDataReady(parentID uint64, req string, resp []byte) {}
func (t *dummyMaster) ChildDataReady(childID uint64, req string, resp []byte) {
	t.logger.Printf("master ChildDataReady, task: %d, epoch: %d, child: %d, ready: %d\n",
		t.taskID, t.epoch, childID, len(t.fromChildren))
	d := new(dummyData)
	json.Unmarshal(resp, d)
	if _, ok := t.fromChildren[childID]; ok {
		return
	}
	t.fromChildren[childID] = d

	// This is a weak form of checking. We can also check the task ids.
	// But this really means that we get all the events from children, we
	// should go into the next epoch now.
	if len(t.fromChildren) == len(t.framework.GetTopology().GetChildren(t.epoch)) {
		for _, g := range t.fromChildren {
			t.gradient.Value += g.Value
		}

		t.dataChan <- t.gradient.Value
		// TODO(xiaoyunwu) We need to do some test here.

		// In real ML, we modify the gradient first. But here it is noop.
		// Notice that we only
		if t.epoch == NumOfIterations {
			t.framework.ShutdownJob()
			close(t.finishChan)
		} else {
			t.logger.Printf("master finished current epoch, task: %d, epoch: %d", t.taskID, t.epoch)
			t.framework.IncEpoch()
		}
	}
}

func (t *dummyMaster) testablyFail(method string, args ...string) bool {
	if t.config == nil {
		return false
	}
	if t.config[method] != "fail" {
		return false
	}
	if len(args) >= 1 && t.config["failepoch"] != "" {
		// we need to care about fail at specific epoch
		if t.config["failepoch"] != args[0] {
			return false
		}
	}
	if !probablyFail(t.config["faillevel"]) {
		return false
	}
	t.logger.Printf("master task %d testably fail, method: %s\n", t.taskID, method)
	t.framework.(*framework).stop()
	t.NodeProducer <- true
	return true
}

// dummySlave is an prototype for data shard in machine learning applications.
// It mainly does to things, pass on parameters to its children, and collect
// gradient back then add them together before make it available to its parent.
type dummySlave struct {
	framework     meritop.Framework
	epoch, taskID uint64
	logger        *log.Logger
	NodeProducer  chan bool
	config        map[string]string

	param, gradient *dummyData
	fromChildren    map[uint64]*dummyData
	gradientReady   *countDownLatch
}

// This is useful to bring the task up to speed from scratch or if it recovers.
func (t *dummySlave) Init(taskID uint64, framework meritop.Framework) {
	t.gradientReady = newCountDownLatch()
	t.taskID = taskID
	t.framework = framework
	t.logger = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile)
	// t.logger = log.New(ioutil.Discard, "", log.Ldate|log.Ltime|log.Lshortfile)
}

// Task need to finish up for exit, last chance to save work?
func (t *dummySlave) Exit() {}

// Ideally, we should also have the following:
func (t *dummySlave) ParentMetaReady(parentID uint64, meta string) {
	t.logger.Printf("slave ParentMetaReady, task: %d, epoch: %d\n", t.taskID, t.epoch)
	t.framework.DataRequest(parentID, meta)
}

func (t *dummySlave) ChildMetaReady(childID uint64, meta string) {
	t.logger.Printf("slave ChildMetaReady, task: %d, epoch: %d\n", t.taskID, t.epoch)
	t.framework.DataRequest(childID, meta)
}

// This give the task an opportunity to cleanup and regroup.
func (t *dummySlave) SetEpoch(epoch uint64) {
	t.logger.Printf("slave SetEpoch, task: %d, epoch: %d\n", t.taskID, epoch)
	t.param = &dummyData{}
	t.gradient = &dummyData{}
	t.gradientReady.ResetCounter(1)

	t.epoch = epoch
	// Make sure we have a clean slate.
	t.fromChildren = make(map[uint64]*dummyData)
}

// These are payload rpc for application purpose.
func (t *dummySlave) ServeAsParent(fromID uint64, req string) []byte {
	b, err := json.Marshal(t.param)
	if err != nil {
		t.logger.Fatalf("Slave can't encode parameter: %v, error: %v\n", t.param, err)
	}
	return b
}

func (t *dummySlave) ServeAsChild(fromID uint64, req string) []byte {
	b, err := json.Marshal(t.gradient)
	if err != nil {
		t.logger.Fatalf("Slave can't encode gradient: %v, error: %v\n", t.gradient, err)
	}
	return b
}

func (t *dummySlave) ParentDataReady(parentID uint64, req string, resp []byte) {
	t.logger.Printf("slave ParentDataReady, task: %d, epoch: %d, parent: %d\n", t.taskID, t.epoch, parentID)
	if t.testablyFail("ParentDataReady") {
		return
	}
	if t.gradientReady.IsDone() {
		return
	}
	t.param = new(dummyData)
	json.Unmarshal(resp, t.param)
	// We need to carry out local compuation.
	t.gradient.Value = t.param.Value * int32(t.framework.GetTaskID())
	t.gradientReady.Done()

	// If this task has children, flag meta so that children can start pull
	// parameter.
	children := t.framework.GetTopology().GetChildren(t.epoch)
	if len(children) != 0 {
		t.framework.FlagMetaToChild("ParamReady")
	} else {
		// On leaf node, we can immediately return by and flag parent
		// that this node is ready.
		t.framework.FlagMetaToParent("GradientReady")
	}
}

func (t *dummySlave) ChildDataReady(childID uint64, req string, resp []byte) {
	t.logger.Printf("slave ChildDataReady, task: %d, epoch: %d, child: %d\n", t.taskID, t.epoch, childID)
	d := new(dummyData)
	json.Unmarshal(resp, d)
	if _, ok := t.fromChildren[childID]; ok {
		return
	}
	t.fromChildren[childID] = d
	// This is a weak form of checking. We can also check the task ids.
	// But this really means that we get all the events from children, we
	// should go into the next epoch now.
	if len(t.fromChildren) == len(t.framework.GetTopology().GetChildren(t.epoch)) {
		// If a new node restart and find out both parent and child meta ready, it will
		// simultaneously request both data. We need to wait until gradient data is there.
		t.gradientReady.Wait()
		// In real ML, we add the gradient first.
		for _, g := range t.fromChildren {
			t.gradient.Value += g.Value
		}

		// If this failure happens, a new node will redo computing again.
		if t.testablyFail("ChildDataReady") {
			return
		}

		t.framework.FlagMetaToParent("GradientReady")

		// if this failure happens, the parent could
		// 1. not have the data yet. In such case, the parent could
		//   1.1 not request the data before a new node restarts. This will cause
		//       double requests since we provide at-least-once semantics.
		//   1.2 request the data with a failed host (request should fail or be
		//       responded with error message).
		// 2. already get the data.
		if t.testablyFail("ChildDataReady") {
			return
		}
	}
}

func (t *dummySlave) testablyFail(method string, args ...string) bool {
	if t.config == nil {
		return false
	}
	if t.config[method] != "fail" {
		return false
	}
	if !probablyFail(t.config["faillevel"]) {
		return false
	}
	t.logger.Printf("slave task %d testably fail, method: %s\n", t.taskID, method)
	t.framework.(*framework).stop()
	t.NodeProducer <- true
	return true
}

func probablyFail(levelStr string) bool {
	level, err := strconv.Atoi(levelStr)
	if err != nil {
		return false
	}
	if level < rand.Intn(100)+1 {
		return false
	}
	return true
}

// used for testing
type SimpleTaskBuilder struct {
	GDataChan    chan int32
	FinishChan   chan struct{}
	NodeProducer chan bool
	MasterConfig map[string]string
	SlaveConfig  map[string]string
}

// This method is called once by framework implementation to get the
// right task implementation for the node/task. It requires the taskID
// for current node, and also a global array of tasks.
func (tc SimpleTaskBuilder) GetTask(taskID uint64) meritop.Task {
	if taskID == 0 {
		return &dummyMaster{
			dataChan:     tc.GDataChan,
			finishChan:   tc.FinishChan,
			NodeProducer: tc.NodeProducer,
			config:       tc.MasterConfig,
		}
	}
	return &dummySlave{
		NodeProducer: tc.NodeProducer,
		config:       tc.SlaveConfig,
	}
}

// I am writing this count down latch because sync.WaitGroup doesn't support
// decrementing counter when it's 0.
type countDownLatch struct {
	sync.Mutex
	cond    *sync.Cond
	counter int
}

func newCountDownLatch() *countDownLatch {
	c := new(countDownLatch)
	c.cond = sync.NewCond(c)
	return c
}

func (c *countDownLatch) ResetCounter(count int) {
	c.counter = count
}

func (c *countDownLatch) IsDone() bool {
	c.Lock()
	defer c.Unlock()
	return c.counter == 0
}

func (c *countDownLatch) Done() {
	c.Lock()
	defer c.Unlock()
	if c.counter == 0 {
		return
	}
	c.counter--
	if c.counter == 0 {
		c.cond.Broadcast()
	}
}

func (c *countDownLatch) Wait() {
	c.Lock()
	defer c.Unlock()
	if c.counter == 0 {
		return
	}
	c.cond.Wait()
}
<commit_msg>change countdown latch to simulate java interface<commit_after>package framework

import (
	"encoding/json"
	"log"
	"math/rand"
	"os"
	"strconv"
	"sync"

	"github.com/go-distributed/meritop"
)

/*
The dummy task is designed for regresion test of meritop framework.
This works with tree topology.
The main idea behind the regression test is following:
There will be two kinds of dummyTasks: master and slaves. We will have one master
sits at the top with taskID = 0, and then rest 6 (2^n - 2) tasks forms a tree under
the master. There will be 10 epochs, from 1 to 10, at each epoch, we send out a
vector with all values equal to epochID, and each slave is supposedly return a vector
with all values equals epochID*taskID, the values are reduced back to master, and
master will print out the epochID and aggregated vector. After all 10 epoch, it kills
job.
*/

const (
	NumOfIterations uint64 = uint64(10)
)

// dummyData is used to carry parameter and gradient;
type dummyData struct {
	Value int32
}

// dummyMaster is prototype of parameter server, for now it does not
// carry out optimization yet. But it should be easy to add support when
// this full tests out.
// Note: in theory, since there should be no parent of this, so we should
// add error checing in the right places. We will skip these test for now.
type dummyMaster struct {
	dataChan      chan int32
	finishChan    chan struct{}
	NodeProducer  chan bool
	framework     meritop.Framework
	epoch, taskID uint64
	logger        *log.Logger
	config        map[string]string

	param, gradient *dummyData
	fromChildren    map[uint64]*dummyData
}

// This is useful to bring the task up to speed from scratch or if it recovers.
func (t *dummyMaster) Init(taskID uint64, framework meritop.Framework) {
	t.taskID = taskID
	t.framework = framework
	t.logger = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile)
	// t.logger = log.New(ioutil.Discard, "", log.Ldate|log.Ltime|log.Lshortfile)
}

// Task need to finish up for exit, last chance to save work?
func (t *dummyMaster) Exit() {}

// Ideally, we should also have the following:
func (t *dummyMaster) ParentMetaReady(parentID uint64, meta string) {}
func (t *dummyMaster) ChildMetaReady(childID uint64, meta string) {
	t.logger.Printf("master ChildMetaReady, task: %d, epoch: %d, child: %d\n", t.taskID, t.epoch, childID)
	// Get data from child. When all the data is back, starts the next epoch.
	t.framework.DataRequest(childID, meta)
}

// This give the task an opportunity to cleanup and regroup.
func (t *dummyMaster) SetEpoch(epoch uint64) {
	t.logger.Printf("master SetEpoch, task: %d, epoch: %d\n", t.taskID, epoch)
	if t.testablyFail("SetEpoch", strconv.FormatUint(epoch, 10)) {
		return
	}

	t.param = &dummyData{}
	t.gradient = &dummyData{}

	t.epoch = epoch
	t.param.Value = int32(t.epoch)

	// Make sure we have a clean slate.
	t.fromChildren = make(map[uint64]*dummyData)
	t.framework.FlagMetaToChild("ParamReady")
}

// These are payload rpc for application purpose.
func (t *dummyMaster) ServeAsParent(fromID uint64, req string) []byte {
	b, err := json.Marshal(t.param)
	if err != nil {
		t.logger.Fatalf("Master can't encode parameter: %v, error: %v\n", t.param, err)
	}
	return b
}

func (t *dummyMaster) ServeAsChild(fromID uint64, req string) []byte {
	return nil
}

func (t *dummyMaster) ParentDataReady(parentID uint64, req string, resp []byte) {}
func (t *dummyMaster) ChildDataReady(childID uint64, req string, resp []byte) {
	t.logger.Printf("master ChildDataReady, task: %d, epoch: %d, child: %d, ready: %d\n",
		t.taskID, t.epoch, childID, len(t.fromChildren))
	d := new(dummyData)
	json.Unmarshal(resp, d)
	if _, ok := t.fromChildren[childID]; ok {
		return
	}
	t.fromChildren[childID] = d

	// This is a weak form of checking. We can also check the task ids.
	// But this really means that we get all the events from children, we
	// should go into the next epoch now.
	if len(t.fromChildren) == len(t.framework.GetTopology().GetChildren(t.epoch)) {
		for _, g := range t.fromChildren {
			t.gradient.Value += g.Value
		}

		t.dataChan <- t.gradient.Value
		// TODO(xiaoyunwu) We need to do some test here.

		// In real ML, we modify the gradient first. But here it is noop.
		// Notice that we only
		if t.epoch == NumOfIterations {
			t.framework.ShutdownJob()
			close(t.finishChan)
		} else {
			t.logger.Printf("master finished current epoch, task: %d, epoch: %d", t.taskID, t.epoch)
			t.framework.IncEpoch()
		}
	}
}

func (t *dummyMaster) testablyFail(method string, args ...string) bool {
	if t.config == nil {
		return false
	}
	if t.config[method] != "fail" {
		return false
	}
	if len(args) >= 1 && t.config["failepoch"] != "" {
		// we need to care about fail at specific epoch
		if t.config["failepoch"] != args[0] {
			return false
		}
	}
	if !probablyFail(t.config["faillevel"]) {
		return false
	}
	t.logger.Printf("master task %d testably fail, method: %s\n", t.taskID, method)
	t.framework.(*framework).stop()
	t.NodeProducer <- true
	return true
}

// dummySlave is an prototype for data shard in machine learning applications.
// It mainly does to things, pass on parameters to its children, and collect
// gradient back then add them together before make it available to its parent.
type dummySlave struct {
	framework     meritop.Framework
	epoch, taskID uint64
	logger        *log.Logger
	NodeProducer  chan bool
	config        map[string]string

	param, gradient *dummyData
	fromChildren    map[uint64]*dummyData
	gradientReady   *countDownLatch
}

// This is useful to bring the task up to speed from scratch or if it recovers.
func (t *dummySlave) Init(taskID uint64, framework meritop.Framework) {
	t.taskID = taskID
	t.framework = framework
	t.logger = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile)
	// t.logger = log.New(ioutil.Discard, "", log.Ldate|log.Ltime|log.Lshortfile)
}

// Task need to finish up for exit, last chance to save work?
func (t *dummySlave) Exit() {}

// Ideally, we should also have the following:
func (t *dummySlave) ParentMetaReady(parentID uint64, meta string) {
	t.logger.Printf("slave ParentMetaReady, task: %d, epoch: %d\n", t.taskID, t.epoch)
	t.framework.DataRequest(parentID, meta)
}

func (t *dummySlave) ChildMetaReady(childID uint64, meta string) {
	t.logger.Printf("slave ChildMetaReady, task: %d, epoch: %d\n", t.taskID, t.epoch)
	t.framework.DataRequest(childID, meta)
}

// This give the task an opportunity to cleanup and regroup.
func (t *dummySlave) SetEpoch(epoch uint64) {
	t.logger.Printf("slave SetEpoch, task: %d, epoch: %d\n", t.taskID, epoch)
	t.param = &dummyData{}
	t.gradient = &dummyData{}
	t.gradientReady = newCountDownLatch(1)

	t.epoch = epoch
	// Make sure we have a clean slate.
	t.fromChildren = make(map[uint64]*dummyData)
}

// These are payload rpc for application purpose.
func (t *dummySlave) ServeAsParent(fromID uint64, req string) []byte {
	b, err := json.Marshal(t.param)
	if err != nil {
		t.logger.Fatalf("Slave can't encode parameter: %v, error: %v\n", t.param, err)
	}
	return b
}

func (t *dummySlave) ServeAsChild(fromID uint64, req string) []byte {
	b, err := json.Marshal(t.gradient)
	if err != nil {
		t.logger.Fatalf("Slave can't encode gradient: %v, error: %v\n", t.gradient, err)
	}
	return b
}

func (t *dummySlave) ParentDataReady(parentID uint64, req string, resp []byte) {
	t.logger.Printf("slave ParentDataReady, task: %d, epoch: %d, parent: %d\n", t.taskID, t.epoch, parentID)
	if t.testablyFail("ParentDataReady") {
		return
	}
	if t.gradientReady.Count() == 0 {
		return
	}
	t.param = new(dummyData)
	json.Unmarshal(resp, t.param)
	// We need to carry out local compuation.
	t.gradient.Value = t.param.Value * int32(t.framework.GetTaskID())
	t.gradientReady.CountDown()

	// If this task has children, flag meta so that children can start pull
	// parameter.
	children := t.framework.GetTopology().GetChildren(t.epoch)
	if len(children) != 0 {
		t.framework.FlagMetaToChild("ParamReady")
	} else {
		// On leaf node, we can immediately return by and flag parent
		// that this node is ready.
		t.framework.FlagMetaToParent("GradientReady")
	}
}

func (t *dummySlave) ChildDataReady(childID uint64, req string, resp []byte) {
	t.logger.Printf("slave ChildDataReady, task: %d, epoch: %d, child: %d\n", t.taskID, t.epoch, childID)
	d := new(dummyData)
	json.Unmarshal(resp, d)
	if _, ok := t.fromChildren[childID]; ok {
		return
	}
	t.fromChildren[childID] = d
	// This is a weak form of checking. We can also check the task ids.
	// But this really means that we get all the events from children, we
	// should go into the next epoch now.
	if len(t.fromChildren) == len(t.framework.GetTopology().GetChildren(t.epoch)) {
		// If a new node restart and find out both parent and child meta ready, it will
		// simultaneously request both data. We need to wait until gradient data is there.
		t.gradientReady.Await()
		// In real ML, we add the gradient first.
		for _, g := range t.fromChildren {
			t.gradient.Value += g.Value
		}

		// If this failure happens, a new node will redo computing again.
		if t.testablyFail("ChildDataReady") {
			return
		}

		t.framework.FlagMetaToParent("GradientReady")

		// if this failure happens, the parent could
		// 1. not have the data yet. In such case, the parent could
		//   1.1 not request the data before a new node restarts. This will cause
		//       double requests since we provide at-least-once semantics.
		//   1.2 request the data with a failed host (request should fail or be
		//       responded with error message).
		// 2. already get the data.
		if t.testablyFail("ChildDataReady") {
			return
		}
	}
}

func (t *dummySlave) testablyFail(method string, args ...string) bool {
	if t.config == nil {
		return false
	}
	if t.config[method] != "fail" {
		return false
	}
	if !probablyFail(t.config["faillevel"]) {
		return false
	}
	t.logger.Printf("slave task %d testably fail, method: %s\n", t.taskID, method)
	t.framework.(*framework).stop()
	t.NodeProducer <- true
	return true
}

func probablyFail(levelStr string) bool {
	level, err := strconv.Atoi(levelStr)
	if err != nil {
		return false
	}
	if level < rand.Intn(100)+1 {
		return false
	}
	return true
}

// used for testing
type SimpleTaskBuilder struct {
	GDataChan    chan int32
	FinishChan   chan struct{}
	NodeProducer chan bool
	MasterConfig map[string]string
	SlaveConfig  map[string]string
}

// This method is called once by framework implementation to get the
// right task implementation for the node/task. It requires the taskID
// for current node, and also a global array of tasks.
func (tc SimpleTaskBuilder) GetTask(taskID uint64) meritop.Task {
	if taskID == 0 {
		return &dummyMaster{
			dataChan:     tc.GDataChan,
			finishChan:   tc.FinishChan,
			NodeProducer: tc.NodeProducer,
			config:       tc.MasterConfig,
		}
	}
	return &dummySlave{
		NodeProducer: tc.NodeProducer,
		config:       tc.SlaveConfig,
	}
}

// I am writing this count down latch because sync.WaitGroup doesn't support
// decrementing counter when it's 0.
type countDownLatch struct {
	sync.Mutex
	cond    *sync.Cond
	counter int
}

func newCountDownLatch(count int) *countDownLatch {
	c := new(countDownLatch)
	c.cond = sync.NewCond(c)
	c.counter = count
	return c
}

func (c *countDownLatch) Count() int {
	c.Lock()
	defer c.Unlock()
	return c.counter
}

func (c *countDownLatch) CountDown() {
	c.Lock()
	defer c.Unlock()
	if c.counter == 0 {
		return
	}
	c.counter--
	if c.counter == 0 {
		c.cond.Broadcast()
	}
}

func (c *countDownLatch) Await() {
	c.Lock()
	defer c.Unlock()
	if c.counter == 0 {
		return
	}
	c.cond.Wait()
}
<|endoftext|>


<commit_before>package brain_test

import (
	"bytes"
	"testing"

	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	"github.com/BytemarkHosting/bytemark-client/lib/output/prettyprint"
)

func TestAPIKeyPrettyPrint(t *testing.T) {
	tests := []struct {
		name  string
		level prettyprint.DetailLevel
		in    brain.APIKey
		out   string
	}{
		{
			name:  "single line",
			level: prettyprint.SingleLine,
			in: brain.APIKey{
				Label: "jeff",
			},
			out: "jeff",
		}, {
			name:  "single line (expired)",
			level: prettyprint.SingleLine,
			in: brain.APIKey{
				Label:     "jeff",
				ExpiresAt: "2006-01-01T01:01:01.000-0000",
			},
			out: "jeff (expired)",
		}, {
			name:  "full no privs expired",
			level: prettyprint.Full,
			in: brain.APIKey{
				Label:     "jeff",
				ExpiresAt: "2006-01-01T01:01:01.0124-0000",
			},
			out: `jeff (expired)
  Expired: 2006-01-01T01:01:01.0124-0000
`,
		}, {
			name:  "full with privs",
			level: prettyprint.Full,
			in: brain.APIKey{
				Label:     "jeff",
				ExpiresAt: "3000-01-01T01:01:01-0000",
				Privileges: brain.Privileges{
					{
						Username:  "jeffathan",
						AccountID: 23,
						Level:     "account_admin",
						APIKeyID:  4,
					},
				},
			},
			out: `jeff
  Expires: 3000-01-01T01:01:01-0000

  Privileges:
    * account_admin on account #23 for jeffathan
`,
		}, {
			name:  "full with key",
			level: prettyprint.Full,
			in: brain.APIKey{
				Label:     "jeff",
				APIKey:    "abcdefgh",
				ExpiresAt: "3006-01-01T01:01:01-0000",
			},
			out: `jeff
  Expires: 3006-01-01T01:01:01-0000
  Key: apikey.abcdefgh
`,
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			buf := bytes.Buffer{}
			err := test.in.PrettyPrint(&buf, test.level)
			if err != nil {
				t.Fatal(err)
			}
			str := buf.String()
			if str != test.out {
				t.Errorf("Output didn't match expected\nexpected: %q\n  actual: %q", test.out, str)
			}
		})
	}

}
<commit_msg>update apikeys test to allow for privileges with account names in them<commit_after>package brain_test

import (
	"bytes"
	"testing"

	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	"github.com/BytemarkHosting/bytemark-client/lib/output/prettyprint"
)

func TestAPIKeyPrettyPrint(t *testing.T) {
	tests := []struct {
		name  string
		level prettyprint.DetailLevel
		in    brain.APIKey
		out   string
	}{
		{
			name:  "single line",
			level: prettyprint.SingleLine,
			in: brain.APIKey{
				Label: "jeff",
			},
			out: "jeff",
		}, {
			name:  "single line (expired)",
			level: prettyprint.SingleLine,
			in: brain.APIKey{
				Label:     "jeff",
				ExpiresAt: "2006-01-01T01:01:01.000-0000",
			},
			out: "jeff (expired)",
		}, {
			name:  "full no privs expired",
			level: prettyprint.Full,
			in: brain.APIKey{
				Label:     "jeff",
				ExpiresAt: "2006-01-01T01:01:01.0124-0000",
			},
			out: `jeff (expired)
  Expired: 2006-01-01T01:01:01.0124-0000
`,
		}, {
			name:  "full with privs",
			level: prettyprint.Full,
			in: brain.APIKey{
				Label:     "jeff",
				ExpiresAt: "3000-01-01T01:01:01-0000",
				Privileges: brain.Privileges{
					{
						Username:    "jeffathan",
						AccountID:   23,
						AccountName: "jeffadiah",
						Level:       "account_admin",
						APIKeyID:    4,
					},
				},
			},
			out: `jeff
  Expires: 3000-01-01T01:01:01-0000

  Privileges:
    * account_admin on account jeffadiah for jeffathan
`,
		}, {
			name:  "full with key",
			level: prettyprint.Full,
			in: brain.APIKey{
				Label:     "jeff",
				APIKey:    "abcdefgh",
				ExpiresAt: "3006-01-01T01:01:01-0000",
			},
			out: `jeff
  Expires: 3006-01-01T01:01:01-0000
  Key: apikey.abcdefgh
`,
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			buf := bytes.Buffer{}
			err := test.in.PrettyPrint(&buf, test.level)
			if err != nil {
				t.Fatal(err)
			}
			str := buf.String()
			if str != test.out {
				t.Errorf("Output didn't match expected\nexpected: %q\n  actual: %q", test.out, str)
			}
		})
	}

}
<|endoftext|>


<commit_before>package model

import (
	"fmt"
	"github.com/jabaraster/go-web-scaffold/src/go/env"
	"github.com/jinzhu/gorm"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
    "github.com/jabaraster/go-web-scaffold/src/go/configuration"
)

var (
	_db *gorm.DB
)

func init() {
    config := configuration.Get().Database
    switch (config.Kind) {
    case configuration.DbKind_SQLite:
        db, err := gorm.Open("sqlite3", config.SQLite.DatabaseFilePath)
        if err != nil {
            panic(err)
        }
        _db = &db
        initializeDatabase()
    case configuration.DbKind_PostgreSQL:
        cs := fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=false",
            env.ResolveEnv(config.PostgreSQL.Host),
            env.ResolveEnv(config.PostgreSQL.Port),
            env.ResolveEnv(config.PostgreSQL.User),
            env.ResolveEnv(config.PostgreSQL.Database),
            env.ResolveEnv(config.PostgreSQL.Password),
        )
        fmt.Println("★", cs)
        db ,err := gorm.Open("postgres", cs)
        if err != nil {
            panic(err)
        }
        _db = &db
        initializeDatabase()
    }
//	if env.DbKind() == env.DbKindSQLite3 {
//		db, err = gorm.Open("sqlite3", "./app.db")
//	} else {
//		cs := fmt.Sprintf("host=%s user=%s dbname=%s password=%s sslmode=%s",
//			env.PostgresHost(),
//			env.PostgresUser(),
//			env.PostgresDbName(),
//			env.PostgresPassword(),
//			env.PostgresSslMode())
//		fmt.Println(cs)
//		db, err = gorm.Open("postgres", cs)
//		if err != nil {
//			panic(err)
//		}
//	}
//	if err != nil {
//		panic(err)
//	}
//	_db = &db
}

func initializeDatabase() {
    _db.LogMode(true)
    _db.CreateTable(&Product{})
    _db.CreateTable(&Order{})
    _db.CreateTable(&AppUser{})
    _db.CreateTable(&AppUserCredential{})
    _db.AutoMigrate(&Product{}, &Order{}, &AppUser{}, &AppUserCredential{})

    createAdminUserIfNecessary()
}

type NotFound interface {
	// nodef
}
type notFoundImpl struct {
	// nodef
}

func NewNotFound() NotFound {
	return notFoundImpl{}
}

type InvalidValue interface {
	GetDescription() string
}

type invalidValue struct {
	description string
}

func (e *invalidValue) GetDescription() string {
	return e.description
}

func NewInvalidValue(description string) InvalidValue {
	return &invalidValue{description: description}
}

func mustInsert(db *gorm.DB, entity interface{}) {
	if err := db.Create(entity).Error; err != nil {
		panic(err)
	}
}
<commit_msg>mainte.<commit_after>package model

import (
	"fmt"
	"github.com/jabaraster/go-web-scaffold/src/go/env"
	"github.com/jinzhu/gorm"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
    "github.com/jabaraster/go-web-scaffold/src/go/configuration"
)

var (
	_db *gorm.DB
)

func init() {
    config := configuration.Get().Database
    switch (config.Kind) {
    case configuration.DbKind_SQLite:
        db, err := gorm.Open("sqlite3", config.SQLite.DatabaseFilePath)
        if err != nil {
            panic(err)
        }
        _db = &db
        initializeDatabase()
    case configuration.DbKind_PostgreSQL:
        cs := fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=disable",
            env.ResolveEnv(config.PostgreSQL.Host),
            env.ResolveEnv(config.PostgreSQL.Port),
            env.ResolveEnv(config.PostgreSQL.User),
            env.ResolveEnv(config.PostgreSQL.Database),
            env.ResolveEnv(config.PostgreSQL.Password),
        )
        fmt.Println("★", cs)
        db ,err := gorm.Open("postgres", cs)
        if err != nil {
            panic(err)
        }
        _db = &db
        initializeDatabase()
    }
//	if env.DbKind() == env.DbKindSQLite3 {
//		db, err = gorm.Open("sqlite3", "./app.db")
//	} else {
//		cs := fmt.Sprintf("host=%s user=%s dbname=%s password=%s sslmode=%s",
//			env.PostgresHost(),
//			env.PostgresUser(),
//			env.PostgresDbName(),
//			env.PostgresPassword(),
//			env.PostgresSslMode())
//		fmt.Println(cs)
//		db, err = gorm.Open("postgres", cs)
//		if err != nil {
//			panic(err)
//		}
//	}
//	if err != nil {
//		panic(err)
//	}
//	_db = &db
}

func initializeDatabase() {
    _db.LogMode(true)
    _db.CreateTable(&Product{})
    _db.CreateTable(&Order{})
    _db.CreateTable(&AppUser{})
    _db.CreateTable(&AppUserCredential{})
    _db.AutoMigrate(&Product{}, &Order{}, &AppUser{}, &AppUserCredential{})

    createAdminUserIfNecessary()
}

type NotFound interface {
	// nodef
}
type notFoundImpl struct {
	// nodef
}

func NewNotFound() NotFound {
	return notFoundImpl{}
}

type InvalidValue interface {
	GetDescription() string
}

type invalidValue struct {
	description string
}

func (e *invalidValue) GetDescription() string {
	return e.description
}

func NewInvalidValue(description string) InvalidValue {
	return &invalidValue{description: description}
}

func mustInsert(db *gorm.DB, entity interface{}) {
	if err := db.Create(entity).Error; err != nil {
		panic(err)
	}
}
<|endoftext|>