<commit_before>// Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
// Use of this source code is governed by an MIT-style license that can be
// found in the LICENSE file.

package scanner

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"
	"code.google.com/p/go.text/unicode/norm"

	"github.com/calmh/syncthing/lamport"
	"github.com/calmh/syncthing/protocol"
)

type Walker struct {
	// Dir is the base directory for the walk
	Dir string
	// BlockSize controls the size of the block used when hashing.
	BlockSize int
	// If IgnoreFile is not empty, it is the name used for the file that holds ignore patterns.
	IgnoreFile string
	// If TempNamer is not nil, it is used to ignore tempory files when walking.
	TempNamer TempNamer
	// If CurrentFiler is not nil, it is queried for the current file before rescanning.
	CurrentFiler CurrentFiler
	// If Suppressor is not nil, it is queried for supression of modified files.
	// Suppressed files will be returned with empty metadata and the Suppressed flag set.
	// Requires CurrentFiler to be set.
	Suppressor Suppressor
	// If IgnorePerms is true, changes to permission bits will not be
	// detected. Scanned files will get zero permission bits and the
	// NoPermissionBits flag set.
	IgnorePerms bool
}

type TempNamer interface {
	// Temporary returns a temporary name for the filed referred to by filepath.
	TempName(path string) string
	// IsTemporary returns true if path refers to the name of temporary file.
	IsTemporary(path string) bool
}

type Suppressor interface {
	// Supress returns true if the update to the named file should be ignored.
	Suppress(name string, fi os.FileInfo) (bool, bool)
}

type CurrentFiler interface {
	// CurrentFile returns the file as seen at last scan.
	CurrentFile(name string) File
}

// Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed.
func (w *Walker) Walk() (files []File, ignore map[string][]string, err error) {
	if debug {
		l.Debugln("Walk", w.Dir, w.BlockSize, w.IgnoreFile)
	}

	err = checkDir(w.Dir)
	if err != nil {
		return
	}

	t0 := time.Now()

	ignore = make(map[string][]string)
	hashFiles := w.walkAndHashFiles(&files, ignore)

	filepath.Walk(w.Dir, w.loadIgnoreFiles(w.Dir, ignore))
	filepath.Walk(w.Dir, hashFiles)

	if debug {
		t1 := time.Now()
		d := t1.Sub(t0).Seconds()
		l.Debugf("Walk in %.02f ms, %.0f files/s", d*1000, float64(len(files))/d)
	}

	err = checkDir(w.Dir)
	return
}

// CleanTempFiles removes all files that match the temporary filename pattern.
func (w *Walker) CleanTempFiles() {
	filepath.Walk(w.Dir, w.cleanTempFile)
}

func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		rn, err := filepath.Rel(dir, p)
		if err != nil {
			return nil
		}

		if pn, sn := filepath.Split(rn); sn == w.IgnoreFile {
			pn := filepath.Clean(pn)
			l.Debugf("pn: %q", pn)
			bs, _ := ioutil.ReadFile(p)
			lines := bytes.Split(bs, []byte("\n"))
			var patterns []string
			for _, line := range lines {
				lineStr := strings.TrimSpace(string(line))
				if len(lineStr) > 0 {
					patterns = append(patterns, lineStr)
				}
			}
			ign[pn] = patterns
		}

		return nil
	}
}

func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			if debug {
				l.Debugln("error:", p, info, err)
			}
			return nil
		}

		rn, err := filepath.Rel(w.Dir, p)
		if err != nil {
			if debug {
				l.Debugln("rel error:", p, err)
			}
			return nil
		}

		if rn == "." {
			return nil
		}

		if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
			// A temporary file
			if debug {
				l.Debugln("temporary:", rn)
			}
			return nil
		}

		if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ign, rn) {
			// An ignored file
			if debug {
				l.Debugln("ignored:", rn)
			}
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if (runtime.GOOS == "linux" || runtime.GOOS == "windows") && !norm.NFC.IsNormalString(rn) {
			l.Warnf("File %q contains non-NFC UTF-8 sequences and cannot be synced. Consider renaming.", rn)
			return nil
		}

		if info.Mode().IsDir() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if cf.Modified == info.ModTime().Unix() && protocol.IsDirectory(cf.Flags) && permUnchanged {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
				} else {
					var flags uint32 = protocol.FlagDirectory
					if w.IgnorePerms {
						flags |= protocol.FlagNoPermBits | 0777
					} else {
						flags |= uint32(info.Mode() & os.ModePerm)
					}
					f := File{
						Name:     rn,
						Version:  lamport.Default.Tick(0),
						Flags:    flags,
						Modified: info.ModTime().Unix(),
					}
					if debug {
						l.Debugln("dir:", cf, f)
					}
					*res = append(*res, f)
				}
				return nil
			}
		}

		if info.Mode().IsRegular() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if !protocol.IsDeleted(cf.Flags) && cf.Modified == info.ModTime().Unix() && permUnchanged {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
					return nil
				}

				if w.Suppressor != nil {
					if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev {
						l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p)
						cf.Suppressed = true
						cf.Version++
						if debug {
							l.Debugln("suppressed:", cf)
						}
						*res = append(*res, cf)
						return nil
					} else if prev && !cur {
						l.Infof("Changes to %q are no longer suppressed.", p)
					}
				}

				if debug {
					l.Debugln("rescan:", cf, info.ModTime().Unix(), info.Mode()&os.ModePerm)
				}
			}

			fd, err := os.Open(p)
			if err != nil {
				if debug {
					l.Debugln("open:", p, err)
				}
				return nil
			}
			defer fd.Close()

			t0 := time.Now()
			blocks, err := Blocks(fd, w.BlockSize)
			if err != nil {
				if debug {
					l.Debugln("hash error:", rn, err)
				}
				return nil
			}
			if debug {
				t1 := time.Now()
				l.Debugln("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s")
			}

			var flags = uint32(info.Mode() & os.ModePerm)
			if w.IgnorePerms {
				flags = protocol.FlagNoPermBits | 0666
			}
			f := File{
				Name:     rn,
				Version:  lamport.Default.Tick(0),
				Size:     info.Size(),
				Flags:    flags,
				Modified: info.ModTime().Unix(),
				Blocks:   blocks,
			}
			*res = append(*res, f)
		}

		return nil
	}
}

func (w *Walker) cleanTempFile(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if info.Mode()&os.ModeType == 0 && w.TempNamer.IsTemporary(path) {
		os.Remove(path)
	}
	return nil
}

func (w *Walker) ignoreFile(patterns map[string][]string, file string) bool {
	first, last := filepath.Split(file)
	for prefix, pats := range patterns {
		if prefix == "." || prefix == first || strings.HasPrefix(first, fmt.Sprintf("%s%c", prefix, os.PathSeparator)) {
			for _, pattern := range pats {
				l.Debugf("%q %q", pattern, last)
				if match, _ := filepath.Match(pattern, last); match {
					return true
				}
			}
		}
	}
	return false
}

func checkDir(dir string) error {
	if info, err := os.Lstat(dir); err != nil {
		return err
	} else if !info.IsDir() {
		return errors.New(dir + ": not a directory")
	} else if debug {
		l.Debugln("checkDir", dir, info)
	}
	return nil
}

func PermsEqual(a, b uint32) bool {
	switch runtime.GOOS {
	case "windows":
		// There is only writeable and read only, represented for user, group
		// and other equally. We only compare against user.
		return a&0600 == b&0600
	default:
		// All bits count
		return a&0777 == b&0777
	}
}
<commit_msg>Remove spurious debug output in .stignore handling<commit_after>// Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
// Use of this source code is governed by an MIT-style license that can be
// found in the LICENSE file.

package scanner

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"
	"code.google.com/p/go.text/unicode/norm"

	"github.com/calmh/syncthing/lamport"
	"github.com/calmh/syncthing/protocol"
)

type Walker struct {
	// Dir is the base directory for the walk
	Dir string
	// BlockSize controls the size of the block used when hashing.
	BlockSize int
	// If IgnoreFile is not empty, it is the name used for the file that holds ignore patterns.
	IgnoreFile string
	// If TempNamer is not nil, it is used to ignore tempory files when walking.
	TempNamer TempNamer
	// If CurrentFiler is not nil, it is queried for the current file before rescanning.
	CurrentFiler CurrentFiler
	// If Suppressor is not nil, it is queried for supression of modified files.
	// Suppressed files will be returned with empty metadata and the Suppressed flag set.
	// Requires CurrentFiler to be set.
	Suppressor Suppressor
	// If IgnorePerms is true, changes to permission bits will not be
	// detected. Scanned files will get zero permission bits and the
	// NoPermissionBits flag set.
	IgnorePerms bool
}

type TempNamer interface {
	// Temporary returns a temporary name for the filed referred to by filepath.
	TempName(path string) string
	// IsTemporary returns true if path refers to the name of temporary file.
	IsTemporary(path string) bool
}

type Suppressor interface {
	// Supress returns true if the update to the named file should be ignored.
	Suppress(name string, fi os.FileInfo) (bool, bool)
}

type CurrentFiler interface {
	// CurrentFile returns the file as seen at last scan.
	CurrentFile(name string) File
}

// Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed.
func (w *Walker) Walk() (files []File, ignore map[string][]string, err error) {
	if debug {
		l.Debugln("Walk", w.Dir, w.BlockSize, w.IgnoreFile)
	}

	err = checkDir(w.Dir)
	if err != nil {
		return
	}

	t0 := time.Now()

	ignore = make(map[string][]string)
	hashFiles := w.walkAndHashFiles(&files, ignore)

	filepath.Walk(w.Dir, w.loadIgnoreFiles(w.Dir, ignore))
	filepath.Walk(w.Dir, hashFiles)

	if debug {
		t1 := time.Now()
		d := t1.Sub(t0).Seconds()
		l.Debugf("Walk in %.02f ms, %.0f files/s", d*1000, float64(len(files))/d)
	}

	err = checkDir(w.Dir)
	return
}

// CleanTempFiles removes all files that match the temporary filename pattern.
func (w *Walker) CleanTempFiles() {
	filepath.Walk(w.Dir, w.cleanTempFile)
}

func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		rn, err := filepath.Rel(dir, p)
		if err != nil {
			return nil
		}

		if pn, sn := filepath.Split(rn); sn == w.IgnoreFile {
			pn := filepath.Clean(pn)
			bs, _ := ioutil.ReadFile(p)
			lines := bytes.Split(bs, []byte("\n"))
			var patterns []string
			for _, line := range lines {
				lineStr := strings.TrimSpace(string(line))
				if len(lineStr) > 0 {
					patterns = append(patterns, lineStr)
				}
			}
			ign[pn] = patterns
		}

		return nil
	}
}

func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			if debug {
				l.Debugln("error:", p, info, err)
			}
			return nil
		}

		rn, err := filepath.Rel(w.Dir, p)
		if err != nil {
			if debug {
				l.Debugln("rel error:", p, err)
			}
			return nil
		}

		if rn == "." {
			return nil
		}

		if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
			// A temporary file
			if debug {
				l.Debugln("temporary:", rn)
			}
			return nil
		}

		if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ign, rn) {
			// An ignored file
			if debug {
				l.Debugln("ignored:", rn)
			}
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if (runtime.GOOS == "linux" || runtime.GOOS == "windows") && !norm.NFC.IsNormalString(rn) {
			l.Warnf("File %q contains non-NFC UTF-8 sequences and cannot be synced. Consider renaming.", rn)
			return nil
		}

		if info.Mode().IsDir() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if cf.Modified == info.ModTime().Unix() && protocol.IsDirectory(cf.Flags) && permUnchanged {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
				} else {
					var flags uint32 = protocol.FlagDirectory
					if w.IgnorePerms {
						flags |= protocol.FlagNoPermBits | 0777
					} else {
						flags |= uint32(info.Mode() & os.ModePerm)
					}
					f := File{
						Name:     rn,
						Version:  lamport.Default.Tick(0),
						Flags:    flags,
						Modified: info.ModTime().Unix(),
					}
					if debug {
						l.Debugln("dir:", cf, f)
					}
					*res = append(*res, f)
				}
				return nil
			}
		}

		if info.Mode().IsRegular() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if !protocol.IsDeleted(cf.Flags) && cf.Modified == info.ModTime().Unix() && permUnchanged {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
					return nil
				}

				if w.Suppressor != nil {
					if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev {
						l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p)
						cf.Suppressed = true
						cf.Version++
						if debug {
							l.Debugln("suppressed:", cf)
						}
						*res = append(*res, cf)
						return nil
					} else if prev && !cur {
						l.Infof("Changes to %q are no longer suppressed.", p)
					}
				}

				if debug {
					l.Debugln("rescan:", cf, info.ModTime().Unix(), info.Mode()&os.ModePerm)
				}
			}

			fd, err := os.Open(p)
			if err != nil {
				if debug {
					l.Debugln("open:", p, err)
				}
				return nil
			}
			defer fd.Close()

			t0 := time.Now()
			blocks, err := Blocks(fd, w.BlockSize)
			if err != nil {
				if debug {
					l.Debugln("hash error:", rn, err)
				}
				return nil
			}
			if debug {
				t1 := time.Now()
				l.Debugln("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s")
			}

			var flags = uint32(info.Mode() & os.ModePerm)
			if w.IgnorePerms {
				flags = protocol.FlagNoPermBits | 0666
			}
			f := File{
				Name:     rn,
				Version:  lamport.Default.Tick(0),
				Size:     info.Size(),
				Flags:    flags,
				Modified: info.ModTime().Unix(),
				Blocks:   blocks,
			}
			*res = append(*res, f)
		}

		return nil
	}
}

func (w *Walker) cleanTempFile(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if info.Mode()&os.ModeType == 0 && w.TempNamer.IsTemporary(path) {
		os.Remove(path)
	}
	return nil
}

func (w *Walker) ignoreFile(patterns map[string][]string, file string) bool {
	first, last := filepath.Split(file)
	for prefix, pats := range patterns {
		if prefix == "." || prefix == first || strings.HasPrefix(first, fmt.Sprintf("%s%c", prefix, os.PathSeparator)) {
			for _, pattern := range pats {
				if match, _ := filepath.Match(pattern, last); match {
					return true
				}
			}
		}
	}
	return false
}

func checkDir(dir string) error {
	if info, err := os.Lstat(dir); err != nil {
		return err
	} else if !info.IsDir() {
		return errors.New(dir + ": not a directory")
	} else if debug {
		l.Debugln("checkDir", dir, info)
	}
	return nil
}

func PermsEqual(a, b uint32) bool {
	switch runtime.GOOS {
	case "windows":
		// There is only writeable and read only, represented for user, group
		// and other equally. We only compare against user.
		return a&0600 == b&0600
	default:
		// All bits count
		return a&0777 == b&0777
	}
}
<|endoftext|>


<commit_before>package pelau

import (
	"net/http"
)

//Response is the interface for manipulating outgoing http information.
type Response interface {

	//Head sends an HTTP header to the client.
	Head(string, string) Response

	//Redirect sends a location header to the client
	Redirect(string, int) Response

	//AddEncoder sets the Encoder that will be used by future calls to WriteData
	AddEncoder(string, Encoder) Response

	//Send  writes out data to the stream but it is first formatted by the current Encoder.
	Send(string, interface{}, func(error, int)) Response

	http.ResponseWriter
}
<commit_msg>Removed Send() and AddEncoder().<commit_after>package pelau

import (
	"net/http"
)

//Response is the interface for manipulating outgoing http information.
type Response interface {

	//Head sends an HTTP header to the client.
	Head(string, string) Response

	//Redirect sends a location header to the client
	Redirect(string, int) Response

	//Stream  writes out data to the stream but it is first formatted by the current Encoder.
	Stream(string, interface{}) (error, int)

	http.ResponseWriter
}
<|endoftext|>


<commit_before>package feature

import (
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc32"
	"net/url"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/model/instance"
	"github.com/cozy/cozy-stack/pkg/config/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
)

type Flags struct {
	DocID   string
	DocRev  string
	M       map[string]interface{}
	Sources []*Flags
}

func (f *Flags) ID() string        { return f.DocID }
func (f *Flags) Rev() string       { return f.DocRev }
func (f *Flags) DocType() string   { return consts.Settings }
func (f *Flags) SetID(id string)   { f.DocID = id }
func (f *Flags) SetRev(rev string) { f.DocRev = rev }
func (f *Flags) Clone() couchdb.Doc {
	clone := Flags{DocID: f.DocID, DocRev: f.DocRev}
	clone.M = make(map[string]interface{})
	for k, v := range f.M {
		clone.M[k] = v
	}
	return &clone
}
func (f *Flags) MarshalJSON() ([]byte, error) {
	return json.Marshal(f.M)
}

func (f *Flags) UnmarshalJSON(bytes []byte) error {
	err := json.Unmarshal(bytes, &f.M)
	if err != nil {
		return err
	}
	if id, ok := f.M["_id"].(string); ok {
		f.SetID(id)
		delete(f.M, "_id")
	}
	if rev, ok := f.M["_rev"].(string); ok {
		f.SetRev(rev)
		delete(f.M, "_rev")
	}
	return nil
}

func GetFlags(inst *instance.Instance) (*Flags, error) {
	sources := make([]*Flags, 0)
	m := make(map[string]interface{})
	flags := &Flags{
		DocID:   consts.FlagsSettingsID,
		M:       m,
		Sources: sources,
	}
	flags.addInstanceFlags(inst)
	if err := flags.addManager(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the manager: %s", err)
	}
	if err := flags.addConfig(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the config: %s", err)
	}
	if err := flags.addContext(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the context: %s", err)
	}
	if err := flags.addDefaults(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the defaults: %s", err)
	}
	return flags, nil
}

func (f *Flags) addInstanceFlags(inst *instance.Instance) {
	if len(inst.FeatureFlags) == 0 {
		return
	}
	m := make(map[string]interface{})
	for k, v := range inst.FeatureFlags {
		m[k] = v
	}
	flags := &Flags{
		DocID: consts.InstanceFlagsSettingsID,
		M:     m,
	}
	f.Sources = append(f.Sources, flags)
	for k, v := range flags.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
}

func (f *Flags) addManager(inst *instance.Instance) error {
	if len(inst.FeatureSets) == 0 {
		return nil
	}
	m, err := getFlagsFromManager(inst)
	if err != nil || len(m) == 0 {
		return err
	}
	flags := &Flags{
		DocID: consts.ManagerFlagsSettingsID,
		M:     m,
	}
	f.Sources = append(f.Sources, flags)
	for k, v := range flags.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
	return nil
}

var (
	cacheDuration      = 12 * time.Hour
	errInvalidResponse = errors.New("Invalid response from the manager")
)

func getFlagsFromManager(inst *instance.Instance) (map[string]interface{}, error) {
	cache := config.GetConfig().CacheStorage
	cacheKey := fmt.Sprintf("flags:%s:%v", inst.ContextName, inst.FeatureSets)
	var flags map[string]interface{}
	if buf, ok := cache.Get(cacheKey); ok {
		if err := json.Unmarshal(buf, &flags); err == nil {
			return flags, nil
		}
	}

	client := instance.APIManagerClient(inst)
	if client == nil {
		return flags, nil
	}
	query := url.Values{
		"sets":    {strings.Join(inst.FeatureSets, ",")},
		"context": {inst.ContextName},
	}.Encode()
	var data map[string]interface{}
	if err := client.Get(fmt.Sprintf("/api/v1/flags?%s", query), &data); err != nil {
		return nil, err
	}
	var ok bool
	if flags, ok = data["flags"].(map[string]interface{}); !ok {
		return nil, errInvalidResponse
	}

	if buf, err := json.Marshal(flags); err == nil {
		cache.Set(cacheKey, buf, cacheDuration)
	}
	return flags, nil
}

func (f *Flags) addConfig(inst *instance.Instance) error {
	ctx, err := inst.SettingsContext()
	if err == instance.ErrContextNotFound {
		return nil
	} else if err != nil {
		return err
	}
	m, ok := ctx["features"].(map[interface{}]interface{})
	if !ok {
		return nil
	}
	normalized := make(map[string]interface{})
	for k, v := range m {
		normalized[fmt.Sprintf("%v", k)] = v
	}
	ctxFlags := &Flags{
		DocID: consts.ConfigFlagsSettingsID,
		M:     normalized,
	}
	f.Sources = append(f.Sources, ctxFlags)
	for k, v := range ctxFlags.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
	return nil
}

func (f *Flags) addContext(inst *instance.Instance) error {
	id := fmt.Sprintf("%s.%s", consts.ContextFlagsSettingsID, inst.ContextName)
	var context Flags
	err := couchdb.GetDoc(couchdb.GlobalDB, consts.Settings, id, &context)
	if couchdb.IsNotFoundError(err) {
		return nil
	} else if err != nil {
		return err
	}
	context.SetID(consts.ContextFlagsSettingsID)
	f.Sources = append(f.Sources, &context)
	for k, v := range context.M {
		if _, ok := f.M[k]; !ok {
			if value := applyRatio(inst, k, v); value != nil {
				f.M[k] = value
			}
		}
	}
	return nil
}

const maxUint32 = 1<<32 - 1

func applyRatio(inst *instance.Instance, key string, data interface{}) interface{} {
	items, ok := data.([]interface{})
	if !ok || len(items) == 0 {
		return nil
	}
	sum := crc32.ChecksumIEEE([]byte(fmt.Sprintf("%s:%s", inst.DocID, key)))
	for i := range items {
		item, ok := items[i].(map[string]interface{})
		if !ok {
			continue
		}
		ratio, ok := item["ratio"].(float64)
		if !ok || ratio == 0.0 {
			continue
		}
		if ratio == 1.0 {
			return item["value"]
		}
		computed := uint32(ratio * maxUint32)
		if computed >= sum {
			return item["value"]
		}
		sum -= computed
	}
	return nil
}

func (f *Flags) addDefaults(inst *instance.Instance) error {
	var defaults Flags
	err := couchdb.GetDoc(couchdb.GlobalDB, consts.Settings, consts.DefaultFlagsSettingsID, &defaults)
	if couchdb.IsNotFoundError(err) {
		return nil
	} else if err != nil {
		return err
	}
	defaults.SetID(consts.DefaultFlagsSettingsID)
	f.Sources = append(f.Sources, &defaults)
	for k, v := range defaults.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
	return nil
}

var _ couchdb.Doc = &Flags{}
<commit_msg>Update the API route for getting flags from the manager<commit_after>package feature

import (
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc32"
	"net/url"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/model/instance"
	"github.com/cozy/cozy-stack/pkg/config/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
)

type Flags struct {
	DocID   string
	DocRev  string
	M       map[string]interface{}
	Sources []*Flags
}

func (f *Flags) ID() string        { return f.DocID }
func (f *Flags) Rev() string       { return f.DocRev }
func (f *Flags) DocType() string   { return consts.Settings }
func (f *Flags) SetID(id string)   { f.DocID = id }
func (f *Flags) SetRev(rev string) { f.DocRev = rev }
func (f *Flags) Clone() couchdb.Doc {
	clone := Flags{DocID: f.DocID, DocRev: f.DocRev}
	clone.M = make(map[string]interface{})
	for k, v := range f.M {
		clone.M[k] = v
	}
	return &clone
}
func (f *Flags) MarshalJSON() ([]byte, error) {
	return json.Marshal(f.M)
}

func (f *Flags) UnmarshalJSON(bytes []byte) error {
	err := json.Unmarshal(bytes, &f.M)
	if err != nil {
		return err
	}
	if id, ok := f.M["_id"].(string); ok {
		f.SetID(id)
		delete(f.M, "_id")
	}
	if rev, ok := f.M["_rev"].(string); ok {
		f.SetRev(rev)
		delete(f.M, "_rev")
	}
	return nil
}

func GetFlags(inst *instance.Instance) (*Flags, error) {
	sources := make([]*Flags, 0)
	m := make(map[string]interface{})
	flags := &Flags{
		DocID:   consts.FlagsSettingsID,
		M:       m,
		Sources: sources,
	}
	flags.addInstanceFlags(inst)
	if err := flags.addManager(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the manager: %s", err)
	}
	if err := flags.addConfig(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the config: %s", err)
	}
	if err := flags.addContext(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the context: %s", err)
	}
	if err := flags.addDefaults(inst); err != nil {
		inst.Logger().WithField("nspace", "flags").
			Warnf("Cannot get the flags from the defaults: %s", err)
	}
	return flags, nil
}

func (f *Flags) addInstanceFlags(inst *instance.Instance) {
	if len(inst.FeatureFlags) == 0 {
		return
	}
	m := make(map[string]interface{})
	for k, v := range inst.FeatureFlags {
		m[k] = v
	}
	flags := &Flags{
		DocID: consts.InstanceFlagsSettingsID,
		M:     m,
	}
	f.Sources = append(f.Sources, flags)
	for k, v := range flags.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
}

func (f *Flags) addManager(inst *instance.Instance) error {
	if len(inst.FeatureSets) == 0 {
		return nil
	}
	m, err := getFlagsFromManager(inst)
	if err != nil || len(m) == 0 {
		return err
	}
	flags := &Flags{
		DocID: consts.ManagerFlagsSettingsID,
		M:     m,
	}
	f.Sources = append(f.Sources, flags)
	for k, v := range flags.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
	return nil
}

var (
	cacheDuration      = 12 * time.Hour
	errInvalidResponse = errors.New("Invalid response from the manager")
)

func getFlagsFromManager(inst *instance.Instance) (map[string]interface{}, error) {
	cache := config.GetConfig().CacheStorage
	cacheKey := fmt.Sprintf("flags:%s:%v", inst.ContextName, inst.FeatureSets)
	var flags map[string]interface{}
	if buf, ok := cache.Get(cacheKey); ok {
		if err := json.Unmarshal(buf, &flags); err == nil {
			return flags, nil
		}
	}

	client := instance.APIManagerClient(inst)
	if client == nil {
		return flags, nil
	}
	query := url.Values{
		"sets":    {strings.Join(inst.FeatureSets, ",")},
		"context": {inst.ContextName},
	}.Encode()
	var data map[string]interface{}
	if err := client.Get(fmt.Sprintf("/api/v1/features?%s", query), &data); err != nil {
		return nil, err
	}
	var ok bool
	if flags, ok = data["flags"].(map[string]interface{}); !ok {
		return nil, errInvalidResponse
	}

	if buf, err := json.Marshal(flags); err == nil {
		cache.Set(cacheKey, buf, cacheDuration)
	}
	return flags, nil
}

func (f *Flags) addConfig(inst *instance.Instance) error {
	ctx, err := inst.SettingsContext()
	if err == instance.ErrContextNotFound {
		return nil
	} else if err != nil {
		return err
	}
	m, ok := ctx["features"].(map[interface{}]interface{})
	if !ok {
		return nil
	}
	normalized := make(map[string]interface{})
	for k, v := range m {
		normalized[fmt.Sprintf("%v", k)] = v
	}
	ctxFlags := &Flags{
		DocID: consts.ConfigFlagsSettingsID,
		M:     normalized,
	}
	f.Sources = append(f.Sources, ctxFlags)
	for k, v := range ctxFlags.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
	return nil
}

func (f *Flags) addContext(inst *instance.Instance) error {
	id := fmt.Sprintf("%s.%s", consts.ContextFlagsSettingsID, inst.ContextName)
	var context Flags
	err := couchdb.GetDoc(couchdb.GlobalDB, consts.Settings, id, &context)
	if couchdb.IsNotFoundError(err) {
		return nil
	} else if err != nil {
		return err
	}
	context.SetID(consts.ContextFlagsSettingsID)
	f.Sources = append(f.Sources, &context)
	for k, v := range context.M {
		if _, ok := f.M[k]; !ok {
			if value := applyRatio(inst, k, v); value != nil {
				f.M[k] = value
			}
		}
	}
	return nil
}

const maxUint32 = 1<<32 - 1

func applyRatio(inst *instance.Instance, key string, data interface{}) interface{} {
	items, ok := data.([]interface{})
	if !ok || len(items) == 0 {
		return nil
	}
	sum := crc32.ChecksumIEEE([]byte(fmt.Sprintf("%s:%s", inst.DocID, key)))
	for i := range items {
		item, ok := items[i].(map[string]interface{})
		if !ok {
			continue
		}
		ratio, ok := item["ratio"].(float64)
		if !ok || ratio == 0.0 {
			continue
		}
		if ratio == 1.0 {
			return item["value"]
		}
		computed := uint32(ratio * maxUint32)
		if computed >= sum {
			return item["value"]
		}
		sum -= computed
	}
	return nil
}

func (f *Flags) addDefaults(inst *instance.Instance) error {
	var defaults Flags
	err := couchdb.GetDoc(couchdb.GlobalDB, consts.Settings, consts.DefaultFlagsSettingsID, &defaults)
	if couchdb.IsNotFoundError(err) {
		return nil
	} else if err != nil {
		return err
	}
	defaults.SetID(consts.DefaultFlagsSettingsID)
	f.Sources = append(f.Sources, &defaults)
	for k, v := range defaults.M {
		if _, ok := f.M[k]; !ok {
			f.M[k] = v
		}
	}
	return nil
}

var _ couchdb.Doc = &Flags{}
<|endoftext|>


<commit_before>package rdb

import (
	"fmt"
	"math/rand"
	"time"

	cdb "github.com/Cepave/open-falcon-backend/common/db"
	"github.com/Cepave/open-falcon-backend/modules/mysqlapi/model"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	"github.com/satori/go.uuid"
)

const (
	defaultTimeout       = 2
	scheduleNameTemplate = "test-schedule-"
	timeThreshold        = 500 * time.Millisecond
	deleteLockSql        = `
		DELETE FROM owl_schedule WHERE sch_name LIKE 'test-schedule-%'
	`
	deleteLogSql = `
		DELETE sl
		FROM owl_schedule sch
		LEFT JOIN owl_schedule_log sl
		ON sch.sch_id = sl.sl_sch_id
		WHERE sch_name LIKE 'test-schedule-%'
	`
)

var _ = Describe("Tests AcquireLock(...)", itSkip.PrependBeforeEach(func() {

	var (
		scheduleName    string
		defaultSchedule *model.Schedule
		defaultNow      time.Time

		/**
		 * Helper function
		 */
		ExpectSuccessSchedule = func(testSchedule *model.Schedule, testError error) {
			Expect(testError).NotTo(HaveOccurred())
			Expect(testSchedule.Uuid).NotTo(Equal(uuid.Nil))
		}

		ExpectLockAndLog = func(expSchedule *model.Schedule, expTime time.Time, expLogCount int) {
			const (
				selectLockSql = `
					SELECT *
					FROM owl_schedule
					WHERE sch_name = ?
				`
				selectLogSql = `
					SELECT *
					FROM owl_schedule_log
					WHERE sl_sch_id = ?
					ORDER BY sl_start_time DESC
					LIMIT 1
				`
				countLogSql = `
					SELECT COUNT(*)
					FROM owl_schedule_log
					WHERE sl_sch_id = ?
				`
			)

			var (
				lockTable model.OwlSchedule
				logTable  model.OwlScheduleLog
			)
			GinkgoT().Log(defaultSchedule)

			By("Check lock")
			DbFacade.SqlxDbCtrl.Get(&lockTable, selectLockSql, expSchedule.Name)
			Expect(lockTable.IsLocked()).To(BeTrue())
			Expect(lockTable.LastUpdateTime).To(BeTemporally("~", expTime, timeThreshold))

			By("Check time")
			DbFacade.SqlxDbCtrl.Get(&logTable, selectLogSql, lockTable.Id)
			Expect(logTable.Timeout).To(Equal(expSchedule.Timeout))
			Expect(logTable.StartTime).To(BeTemporally("~", expTime, timeThreshold))

			By("Check log count")
			var count int
			DbFacade.SqlxDbCtrl.Get(&count, countLogSql, lockTable.Id)
			Expect(count).To(Equal(expLogCount))
		}
		// :~)
	)

	BeforeEach(func() {
		scheduleName = scheduleNameTemplate + fmt.Sprint(rand.Int())
		defaultSchedule = model.NewSchedule(scheduleName, defaultTimeout)
		defaultNow = time.Now()
	})

	AfterEach(func() {
		inTx(deleteLogSql, deleteLockSql)
	})

	Context("Schedule is new", func() {
		It("should acquire the lock", func() {
			err := AcquireLock(defaultSchedule, defaultNow)

			ExpectSuccessSchedule(defaultSchedule, err)
			ExpectLockAndLog(defaultSchedule, defaultNow, 1)
		})
	})

	Context("A schedule has been created", func() {
		JustBeforeEach(func() {
			_ = AcquireLock(defaultSchedule, defaultNow)
		})

		Context("lock is held too long", func() {
			It("should preempt the lock", func() {
				thisTimeout := defaultTimeout + 1
				newCurrent := defaultNow.Add(time.Duration(thisTimeout) * time.Second)
				ps := model.NewSchedule(scheduleName, thisTimeout)
				err := AcquireLock(ps, newCurrent)

				ExpectSuccessSchedule(ps, err)
				ExpectLockAndLog(ps, newCurrent, 2)
			})
		})

		Context("lock is just held", func() {
			It("should trigger error", func() {
				thisTimeout := defaultTimeout + 1
				ps := model.NewSchedule(scheduleName, thisTimeout)
				err := AcquireLock(ps, defaultNow)

				Expect(err).To(HaveOccurred())
				Expect(ps.Uuid).To(Equal(uuid.Nil))

				ExpectLockAndLog(defaultSchedule, defaultNow, 1)
			})
		})

		Context("lock is held but cannot determine the timeout", func() {
			BeforeEach(func() {
				_ = AcquireLock(defaultSchedule, defaultNow)
			})

			JustBeforeEach(func() {
				inTx(deleteLogSql)
			})

			It("should preempt the lock", func() {
				By("Acquire lock from the crashed task")
				thisTimeout := defaultTimeout + 1
				newCurrent := defaultNow.Add(time.Duration(thisTimeout) * time.Second)
				sp := model.NewSchedule(scheduleName, thisTimeout)
				err := AcquireLock(sp, newCurrent)

				ExpectSuccessSchedule(sp, err)
				ExpectLockAndLog(sp, newCurrent, 1)
			})
		})

	})

}))

var _ = Describe("Tests FreeLock(...)", itSkip.PrependBeforeEach(func() {

	var (
		scheduleName    string
		defaultSchedule *model.Schedule
		defaultNow      time.Time
		defaultErrMsg   string = "Default error message."
	)

	BeforeEach(func() {
		scheduleName = scheduleNameTemplate + fmt.Sprint(rand.Int())
		defaultSchedule = model.NewSchedule(scheduleName, defaultTimeout)
		defaultNow = time.Now()
	})

	// AfterEach(func() {
	// 	inTx(deleteLogSql, deleteLockSql)
	// })

	JustBeforeEach(func() {
		_ = AcquireLock(defaultSchedule, defaultNow)
		GinkgoT().Log(defaultSchedule)
	})

	DescribeTable("Free lock & record log",
		func(expStatus model.TaskStatus, expMsg *string) {
			var (
				expSchedule = defaultSchedule
				expTime     = defaultNow.Add(time.Second)

				lockTable model.OwlSchedule
				logTable  model.OwlScheduleLog
			)
			FreeLock(expSchedule, expStatus, expMsg, expTime)

			By("Check lock")
			DbFacade.SqlxDbCtrl.Get(&lockTable, `
				SELECT *
				FROM owl_schedule
				WHERE sch_name = ?
			`, expSchedule.Name)
			Expect(lockTable.IsLocked()).To(BeFalse())
			Expect(lockTable.LastUpdateTime).To(BeTemporally("~", expTime, timeThreshold))

			By("Check log")
			uuid := cdb.DbUuid(expSchedule.Uuid)
			DbFacade.SqlxDbCtrl.Get(&logTable, `
				SELECT *
				FROM owl_schedule_log
				WHERE sl_uuid = ?
			`, uuid)
			Expect(logTable.Status).To(Equal(byte(expStatus)))
			Expect(logTable.Message).To(Equal(expMsg))
			Expect(*logTable.EndTime).To(BeTemporally("~", expTime, timeThreshold))
		},
		Entry("DONE", model.DONE, nil),
		Entry("FAIL", model.FAIL, &defaultErrMsg),
	)
}))
<commit_msg>[OWL-1997] Uncomment clean-up sql.<commit_after>package rdb

import (
	"fmt"
	"math/rand"
	"time"

	cdb "github.com/Cepave/open-falcon-backend/common/db"
	"github.com/Cepave/open-falcon-backend/modules/mysqlapi/model"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	"github.com/satori/go.uuid"
)

const (
	defaultTimeout       = 2
	scheduleNameTemplate = "test-schedule-"
	timeThreshold        = 500 * time.Millisecond
	deleteLockSql        = `
		DELETE FROM owl_schedule WHERE sch_name LIKE 'test-schedule-%'
	`
	deleteLogSql = `
		DELETE sl
		FROM owl_schedule sch
		LEFT JOIN owl_schedule_log sl
		ON sch.sch_id = sl.sl_sch_id
		WHERE sch_name LIKE 'test-schedule-%'
	`
)

var _ = Describe("Tests AcquireLock(...)", itSkip.PrependBeforeEach(func() {

	var (
		scheduleName    string
		defaultSchedule *model.Schedule
		defaultNow      time.Time

		/**
		 * Helper function
		 */
		ExpectSuccessSchedule = func(testSchedule *model.Schedule, testError error) {
			Expect(testError).NotTo(HaveOccurred())
			Expect(testSchedule.Uuid).NotTo(Equal(uuid.Nil))
		}

		ExpectLockAndLog = func(expSchedule *model.Schedule, expTime time.Time, expLogCount int) {
			const (
				selectLockSql = `
					SELECT *
					FROM owl_schedule
					WHERE sch_name = ?
				`
				selectLogSql = `
					SELECT *
					FROM owl_schedule_log
					WHERE sl_sch_id = ?
					ORDER BY sl_start_time DESC
					LIMIT 1
				`
				countLogSql = `
					SELECT COUNT(*)
					FROM owl_schedule_log
					WHERE sl_sch_id = ?
				`
			)

			var (
				lockTable model.OwlSchedule
				logTable  model.OwlScheduleLog
			)
			GinkgoT().Log(defaultSchedule)

			By("Check lock")
			DbFacade.SqlxDbCtrl.Get(&lockTable, selectLockSql, expSchedule.Name)
			Expect(lockTable.IsLocked()).To(BeTrue())
			Expect(lockTable.LastUpdateTime).To(BeTemporally("~", expTime, timeThreshold))

			By("Check time")
			DbFacade.SqlxDbCtrl.Get(&logTable, selectLogSql, lockTable.Id)
			Expect(logTable.Timeout).To(Equal(expSchedule.Timeout))
			Expect(logTable.StartTime).To(BeTemporally("~", expTime, timeThreshold))

			By("Check log count")
			var count int
			DbFacade.SqlxDbCtrl.Get(&count, countLogSql, lockTable.Id)
			Expect(count).To(Equal(expLogCount))
		}
		// :~)
	)

	BeforeEach(func() {
		scheduleName = scheduleNameTemplate + fmt.Sprint(rand.Int())
		defaultSchedule = model.NewSchedule(scheduleName, defaultTimeout)
		defaultNow = time.Now()
	})

	AfterEach(func() {
		inTx(deleteLogSql, deleteLockSql)
	})

	Context("Schedule is new", func() {
		It("should acquire the lock", func() {
			err := AcquireLock(defaultSchedule, defaultNow)

			ExpectSuccessSchedule(defaultSchedule, err)
			ExpectLockAndLog(defaultSchedule, defaultNow, 1)
		})
	})

	Context("A schedule has been created", func() {
		JustBeforeEach(func() {
			_ = AcquireLock(defaultSchedule, defaultNow)
		})

		Context("lock is held too long", func() {
			It("should preempt the lock", func() {
				thisTimeout := defaultTimeout + 1
				newCurrent := defaultNow.Add(time.Duration(thisTimeout) * time.Second)
				ps := model.NewSchedule(scheduleName, thisTimeout)
				err := AcquireLock(ps, newCurrent)

				ExpectSuccessSchedule(ps, err)
				ExpectLockAndLog(ps, newCurrent, 2)
			})
		})

		Context("lock is just held", func() {
			It("should trigger error", func() {
				thisTimeout := defaultTimeout + 1
				ps := model.NewSchedule(scheduleName, thisTimeout)
				err := AcquireLock(ps, defaultNow)

				Expect(err).To(HaveOccurred())
				Expect(ps.Uuid).To(Equal(uuid.Nil))

				ExpectLockAndLog(defaultSchedule, defaultNow, 1)
			})
		})

		Context("lock is held but cannot determine the timeout", func() {
			BeforeEach(func() {
				_ = AcquireLock(defaultSchedule, defaultNow)
			})

			JustBeforeEach(func() {
				inTx(deleteLogSql)
			})

			It("should preempt the lock", func() {
				By("Acquire lock from the crashed task")
				thisTimeout := defaultTimeout + 1
				newCurrent := defaultNow.Add(time.Duration(thisTimeout) * time.Second)
				sp := model.NewSchedule(scheduleName, thisTimeout)
				err := AcquireLock(sp, newCurrent)

				ExpectSuccessSchedule(sp, err)
				ExpectLockAndLog(sp, newCurrent, 1)
			})
		})

	})

}))

var _ = Describe("Tests FreeLock(...)", itSkip.PrependBeforeEach(func() {

	var (
		scheduleName    string
		defaultSchedule *model.Schedule
		defaultNow      time.Time
		defaultErrMsg   string = "Default error message."
	)

	BeforeEach(func() {
		scheduleName = scheduleNameTemplate + fmt.Sprint(rand.Int())
		defaultSchedule = model.NewSchedule(scheduleName, defaultTimeout)
		defaultNow = time.Now()
	})

	AfterEach(func() {
		inTx(deleteLogSql, deleteLockSql)
	})

	JustBeforeEach(func() {
		_ = AcquireLock(defaultSchedule, defaultNow)
		GinkgoT().Log(defaultSchedule)
	})

	DescribeTable("Free lock & record log",
		func(expStatus model.TaskStatus, expMsg *string) {
			var (
				expSchedule = defaultSchedule
				expTime     = defaultNow.Add(time.Second)

				lockTable model.OwlSchedule
				logTable  model.OwlScheduleLog
			)
			FreeLock(expSchedule, expStatus, expMsg, expTime)

			By("Check lock")
			DbFacade.SqlxDbCtrl.Get(&lockTable, `
				SELECT *
				FROM owl_schedule
				WHERE sch_name = ?
			`, expSchedule.Name)
			Expect(lockTable.IsLocked()).To(BeFalse())
			Expect(lockTable.LastUpdateTime).To(BeTemporally("~", expTime, timeThreshold))

			By("Check log")
			uuid := cdb.DbUuid(expSchedule.Uuid)
			DbFacade.SqlxDbCtrl.Get(&logTable, `
				SELECT *
				FROM owl_schedule_log
				WHERE sl_uuid = ?
			`, uuid)
			Expect(logTable.Status).To(Equal(byte(expStatus)))
			Expect(logTable.Message).To(Equal(expMsg))
			Expect(*logTable.EndTime).To(BeTemporally("~", expTime, timeThreshold))
		},
		Entry("DONE", model.DONE, nil),
		Entry("FAIL", model.FAIL, &defaultErrMsg),
	)
}))
<|endoftext|>


<commit_before>/*Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package models

import (
	"github.com/skyrings/skyring-common/monitoring"
	"github.com/skyrings/skyring-common/tools/uuid"
	"time"
)

type Node struct {
	NodeId        uuid.UUID         `json:"nodeid"`
	Hostname      string            `json:"hostname"`
	Tags          []string          `json:"tags"`
	ManagementIP4 string            `json:"management_ip4"`
	ClusterIP4    string            `json:"cluster_ip4"`
	PublicIP4     string            `json:"public_ip4"`
	ClusterId     uuid.UUID         `json:"clusterid"`
	Location      string            `json:"location"`
	Status        NodeStatus        `json:"status"`
	State         NodeState         `json:"state"`
	AlmStatus     AlarmStatus       `json:"almstatus"`
	AlmCount      int               `json:"almcount"`
	Options       map[string]string `json:"options"`
	CPUs          []Cpu             `json:"cpus"`
	NetworkInfo   Network           `json:"network_info"`
	StorageDisks  []Disk            `json:"storage_disks"`
	Memory        Memory            `json:"memory"`
	OS            OperatingSystem   `json:"os"`
	Enabled       bool              `json:"enabled"`
}

type Network struct {
	IPv4   []string `bson:"ipv4",json:"ipv4"`     // TODO: use ipv4 type
	IPv6   []string `bson:"ipv6",json:"ipv6"`     // TODO: use ipv6 type
	Subnet []string `bson:"subnet",json:"subnet"` // TODO: use subnet type
}

type Disk struct {
	DevName        string    `bson:"devname",json:"devname"`
	FSType         string    `bson:"fstype",json:"fstype"`
	FSUUID         uuid.UUID `bson:"fsuuid",json:"fsuuid"`
	Model          string    `bson:"model",json:"model"`
	MountPoint     []string  `bson:"mountpoint",json:"mountpoint"`
	Name           string    `bson:"name",json:"name"`
	Parent         string    `bson:"parent",json:"parent"`
	Size           uint64    `bson:"size",json:"size"`
	Type           string    `bson:"type",json:"type"`
	Used           bool      `bson:"used",json:"used"`
	SSD            bool      `bson:"ssd",json:"ssd"`
	Vendor         string    `bson:"vendor",json:"vendor"`
	StorageProfile string    `bson:"storageprofile",json:"storageprofile"`
	DiskId         uuid.UUID `bson:"diskid",json:"diskid"`
}

type Cpu struct {
	Architecture   string `bson:"architecture",json:"architecture"`
	CpuOpMode      string `bson:"cpuopmode",json:"cpuopmode"`
	CPUs           string `bson:"cpus",json:"cpus"`
	VendorId       string `bson:"vendorid",json:"vendorid"`
	ModelName      string `bson:"modelname",json:"modelname"`
	CPUFamily      string `bson:"cpufamily",json:"cpufamily"`
	CPUMHz         string `bson:"cpumhz",json:"cpumhz"`
	Model          string `bson:"model",json:"model"`
	CoresPerSocket string `bson:"corespersocket",json:"corespersocket"`
}

type OperatingSystem struct {
	Name          string `bson:"name",json:"name"`
	OSVersion     string `bson:"osversion",json:"osversion"`
	KernelVersion string `bson:"kernelversion",json:"kernelversion"`
	SELinuxMode   string `bson:"selinuxmode",json:"selinuxmode"`
}

type Memory struct {
	TotalSize string `bson:"totalsize",json:"totalsize"`
	SwapTotal string `bson:"swaptotal",json:"swaptotal"`
	Active    string `bson:"active",json:"active"`
	Type      string `bson:"type",json:"type"`
}

type User struct {
	Username            string   `json:"username"`
	Email               string   `json:"email"`
	Hash                []byte   `json:"hash"`
	Role                string   `json:"role"`
	Groups              []string `json:"groups"`
	Type                int      `json:"type"`
	Status              bool     `json:"status"`
	FirstName           string   `json:"firstname"`
	LastName            string   `json:"lastname"`
	NotificationEnabled bool     `json:"notificationenabled"`
}

type Cluster struct {
	ClusterId          uuid.UUID         `json:"clusterid"`
	Name               string            `json:"name"`
	CompatVersion      string            `json:"compat_version"`
	Type               string            `json:"type"`
	WorkLoad           string            `json:"workload"`
	Status             ClusterStatus     `json:"status"`
	Tags               []string          `json:"tags"`
	Options            map[string]string `json:"options"`
	OpenStackServices  []string          `json:"openstack_services"`
	Networks           ClusterNetworks   `json:"networks"`
	Monitoring         MonitoringState   `json:"monitoring"`
	MonitoringInterval int               `json:"monitoringinterval"`
	State              ClusterState      `json:"state"`
	AlmStatus          AlarmStatus       `json:"almstatus"`
	AlmCount           int               `json:"almcount"`
}

type MonitoringState struct {
	Plugins    []monitoring.Plugin `json:"plugins"`
	StaleNodes []string            `json:"stalenodes"`
}

type ClusterNetworks struct {
	Cluster string `json:"cluster"`
	Public  string `json:"public"`
}

type StorageLogicalUnit struct {
	SluId             uuid.UUID         `json:"sluid"`
	Name              string            `json:"name"`
	Type              int               `json:"type"`
	ClusterId         uuid.UUID         `json:"clusterid"`
	NodeId            uuid.UUID         `json:"nodeid"`
	StorageId         uuid.UUID         `json:"storageid"`
	StorageDeviceId   uuid.UUID         `json:"storagedeviceid"`
	StorageDeviceSize uint64            `json:"storagedevicesize"`
	Status            string            `json:"status"`
	Options           map[string]string `json:"options"`
	StorageProfile    string            `json:"storageprofile"`
}

type Storage struct {
	StorageId           uuid.UUID         `json:"storageid"`
	Name                string            `json:"name"`
	Type                string            `json:"type"`
	Tags                []string          `json:"tags"`
	ClusterId           uuid.UUID         `json:"clusterid"`
	Size                string            `json:"size"`
	Status              string            `json:"status"`
	Replicas            int               `json:"replicas"`
	Profile             string            `json:"profile"`
	SnapshotsEnabled    bool              `json:"snapshots_enabled"`
	SnapshotScheduleIds []uuid.UUID       `json:"snapshot_schedule_ids"`
	QuotaEnabled        bool              `json:"quota_enabled"`
	QuotaParams         map[string]string `json:"quota_params"`
	Options             map[string]string `json:"options"`
}

type SnapshotSchedule struct {
	Id            uuid.UUID `json:"id"`
	Recurrence    string    `json:"recurrence"`
	Interval      int       `json:"interval"`
	ExecutionTime string    `json:"execution_time"`
	Days          []string  `json:"days"`
	StartFrom     string    `json:"start_from"`
	EndBy         string    `json:"endby"`
}

type Status struct {
	Timestamp time.Time
	Message   string
}

type AppTask struct {
	Id         uuid.UUID         `json:"id"`
	Name       string            `json:"name"`
	ParentId   uuid.UUID         `json:"parentid"`
	Started    bool              `json:"started"`
	Completed  bool              `json:"completed"`
	StatusList []Status          `json:"statuslist"`
	Tag        map[string]string `json:"tag"`
	SubTasks   []uuid.UUID       `json:"subtasks"`
	Status     TaskStatus        `json:"status"`
}

type DiskProfile struct {
	Type  DiskType `json:"disktype"`
	Speed int      `json:"speed"`
}

type StorageProfile struct {
	Name     string      `json:"name"`
	Rule     DiskProfile `json:"rule"`
	Priority int         `json:"priority"`
	Default  bool        `json:"default"`
}

type ExternalUsers struct {
	Users      []User
	TotalCount int
	StartIndex int
	EndIndex   int
}

type Directory struct {
	LdapServer  string
	Port        uint
	Base        string
	DomainAdmin string
	Password    string
	Uid         string
	FirstName   string
	LastName    string
	DisplayName string
	Email       string
}
<commit_msg>Models:Fingerprint added in Node structure<commit_after>/*Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package models

import (
	"github.com/skyrings/skyring-common/monitoring"
	"github.com/skyrings/skyring-common/tools/uuid"
	"time"
)

type Node struct {
	NodeId        uuid.UUID         `json:"nodeid"`
	Hostname      string            `json:"hostname"`
	Tags          []string          `json:"tags"`
	ManagementIP4 string            `json:"management_ip4"`
	ClusterIP4    string            `json:"cluster_ip4"`
	PublicIP4     string            `json:"public_ip4"`
	ClusterId     uuid.UUID         `json:"clusterid"`
	Location      string            `json:"location"`
	Status        NodeStatus        `json:"status"`
	State         NodeState         `json:"state"`
	AlmStatus     AlarmStatus       `json:"almstatus"`
	AlmCount      int               `json:"almcount"`
	Options       map[string]string `json:"options"`
	CPUs          []Cpu             `json:"cpus"`
	NetworkInfo   Network           `json:"network_info"`
	StorageDisks  []Disk            `json:"storage_disks"`
	Memory        Memory            `json:"memory"`
	OS            OperatingSystem   `json:"os"`
	Enabled       bool              `json:"enabled"`
	Fingerprint   string		`json:"saltfingerprint"`
}

type Network struct {
	IPv4   []string `bson:"ipv4",json:"ipv4"`     // TODO: use ipv4 type
	IPv6   []string `bson:"ipv6",json:"ipv6"`     // TODO: use ipv6 type
	Subnet []string `bson:"subnet",json:"subnet"` // TODO: use subnet type
}

type Disk struct {
	DevName        string    `bson:"devname",json:"devname"`
	FSType         string    `bson:"fstype",json:"fstype"`
	FSUUID         uuid.UUID `bson:"fsuuid",json:"fsuuid"`
	Model          string    `bson:"model",json:"model"`
	MountPoint     []string  `bson:"mountpoint",json:"mountpoint"`
	Name           string    `bson:"name",json:"name"`
	Parent         string    `bson:"parent",json:"parent"`
	Size           uint64    `bson:"size",json:"size"`
	Type           string    `bson:"type",json:"type"`
	Used           bool      `bson:"used",json:"used"`
	SSD            bool      `bson:"ssd",json:"ssd"`
	Vendor         string    `bson:"vendor",json:"vendor"`
	StorageProfile string    `bson:"storageprofile",json:"storageprofile"`
	DiskId         uuid.UUID `bson:"diskid",json:"diskid"`
}

type Cpu struct {
	Architecture   string `bson:"architecture",json:"architecture"`
	CpuOpMode      string `bson:"cpuopmode",json:"cpuopmode"`
	CPUs           string `bson:"cpus",json:"cpus"`
	VendorId       string `bson:"vendorid",json:"vendorid"`
	ModelName      string `bson:"modelname",json:"modelname"`
	CPUFamily      string `bson:"cpufamily",json:"cpufamily"`
	CPUMHz         string `bson:"cpumhz",json:"cpumhz"`
	Model          string `bson:"model",json:"model"`
	CoresPerSocket string `bson:"corespersocket",json:"corespersocket"`
}

type OperatingSystem struct {
	Name          string `bson:"name",json:"name"`
	OSVersion     string `bson:"osversion",json:"osversion"`
	KernelVersion string `bson:"kernelversion",json:"kernelversion"`
	SELinuxMode   string `bson:"selinuxmode",json:"selinuxmode"`
}

type Memory struct {
	TotalSize string `bson:"totalsize",json:"totalsize"`
	SwapTotal string `bson:"swaptotal",json:"swaptotal"`
	Active    string `bson:"active",json:"active"`
	Type      string `bson:"type",json:"type"`
}

type User struct {
	Username            string   `json:"username"`
	Email               string   `json:"email"`
	Hash                []byte   `json:"hash"`
	Role                string   `json:"role"`
	Groups              []string `json:"groups"`
	Type                int      `json:"type"`
	Status              bool     `json:"status"`
	FirstName           string   `json:"firstname"`
	LastName            string   `json:"lastname"`
	NotificationEnabled bool     `json:"notificationenabled"`
}

type Cluster struct {
	ClusterId          uuid.UUID         `json:"clusterid"`
	Name               string            `json:"name"`
	CompatVersion      string            `json:"compat_version"`
	Type               string            `json:"type"`
	WorkLoad           string            `json:"workload"`
	Status             ClusterStatus     `json:"status"`
	Tags               []string          `json:"tags"`
	Options            map[string]string `json:"options"`
	OpenStackServices  []string          `json:"openstack_services"`
	Networks           ClusterNetworks   `json:"networks"`
	Monitoring         MonitoringState   `json:"monitoring"`
	MonitoringInterval int               `json:"monitoringinterval"`
	State              ClusterState      `json:"state"`
	AlmStatus          AlarmStatus       `json:"almstatus"`
	AlmCount           int               `json:"almcount"`
}

type MonitoringState struct {
	Plugins    []monitoring.Plugin `json:"plugins"`
	StaleNodes []string            `json:"stalenodes"`
}

type ClusterNetworks struct {
	Cluster string `json:"cluster"`
	Public  string `json:"public"`
}

type StorageLogicalUnit struct {
	SluId             uuid.UUID         `json:"sluid"`
	Name              string            `json:"name"`
	Type              int               `json:"type"`
	ClusterId         uuid.UUID         `json:"clusterid"`
	NodeId            uuid.UUID         `json:"nodeid"`
	StorageId         uuid.UUID         `json:"storageid"`
	StorageDeviceId   uuid.UUID         `json:"storagedeviceid"`
	StorageDeviceSize uint64            `json:"storagedevicesize"`
	Status            string            `json:"status"`
	Options           map[string]string `json:"options"`
	StorageProfile    string            `json:"storageprofile"`
}

type Storage struct {
	StorageId           uuid.UUID         `json:"storageid"`
	Name                string            `json:"name"`
	Type                string            `json:"type"`
	Tags                []string          `json:"tags"`
	ClusterId           uuid.UUID         `json:"clusterid"`
	Size                string            `json:"size"`
	Status              string            `json:"status"`
	Replicas            int               `json:"replicas"`
	Profile             string            `json:"profile"`
	SnapshotsEnabled    bool              `json:"snapshots_enabled"`
	SnapshotScheduleIds []uuid.UUID       `json:"snapshot_schedule_ids"`
	QuotaEnabled        bool              `json:"quota_enabled"`
	QuotaParams         map[string]string `json:"quota_params"`
	Options             map[string]string `json:"options"`
}

type SnapshotSchedule struct {
	Id            uuid.UUID `json:"id"`
	Recurrence    string    `json:"recurrence"`
	Interval      int       `json:"interval"`
	ExecutionTime string    `json:"execution_time"`
	Days          []string  `json:"days"`
	StartFrom     string    `json:"start_from"`
	EndBy         string    `json:"endby"`
}

type Status struct {
	Timestamp time.Time
	Message   string
}

type AppTask struct {
	Id         uuid.UUID         `json:"id"`
	Name       string            `json:"name"`
	ParentId   uuid.UUID         `json:"parentid"`
	Started    bool              `json:"started"`
	Completed  bool              `json:"completed"`
	StatusList []Status          `json:"statuslist"`
	Tag        map[string]string `json:"tag"`
	SubTasks   []uuid.UUID       `json:"subtasks"`
	Status     TaskStatus        `json:"status"`
}

type DiskProfile struct {
	Type  DiskType `json:"disktype"`
	Speed int      `json:"speed"`
}

type StorageProfile struct {
	Name     string      `json:"name"`
	Rule     DiskProfile `json:"rule"`
	Priority int         `json:"priority"`
	Default  bool        `json:"default"`
}

type ExternalUsers struct {
	Users      []User
	TotalCount int
	StartIndex int
	EndIndex   int
}

type Directory struct {
	LdapServer  string
	Port        uint
	Base        string
	DomainAdmin string
	Password    string
	Uid         string
	FirstName   string
	LastName    string
	DisplayName string
	Email       string
}
<|endoftext|>


<commit_before>package scraper

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"
)

// CustomClient is the state for our custom http wrapper, which houses
// the needed data to be able to rewrite the outgoing request during
// redirects.
type CustomClient struct {
	URL       string
	IP        string
	Host      string
	ResultURL *url.URL // represents the url for the resulting request, without modifications
	OriginURL *url.URL // represents the url from the original request, without modifications
}

// CustomResponse is the wrapped response from http.Client.Do() which also
// includes a timer of how long the request took, and a few other minor
// extras.
type CustomResponse struct {
	*http.Response
	Time *TimerResult
	URL  string
}

var reIP = regexp.MustCompile(`^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$`)

func (c *CustomClient) redirectHandler(req *http.Request, via []*http.Request) error {
	c.requestWrap(req)

	// rewrite Referer (Referrer) if it exists, to have the proper hostname
	uri := via[len(via)-1].URL
	uri.Host = via[len(via)-1].Host
	req.Header.Set("Referer", uri.String())

	if len(via) > 3 {
		// assume too many redirects
		return errors.New("too many redirects (3)")
	}

	if reIP.MatchString(req.Host) && req.Host != c.IP {
		return errors.New("Redirected to IP that doesn't match proxy")
	}

	cHost := strings.ToLower(req.URL.Host)
	oHost := strings.ToLower(c.Host)
	if cHost != oHost && cHost != "www."+oHost && "www."+cHost != oHost {
		if c.OriginURL.Path == "" {
			return errors.New("Redirection does not match origin host")
		}
	}

	return nil
}

func (c *CustomClient) requestWrap(req *http.Request) *http.Request {
	// spoof useragent, as there are going to be sites/servers that are
	// setup to deny by a specific useragent string (or lack there of)
	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36")

	// if an IP address is provided, rewrite the Host headers
	// of note: if we plan to support custom ports, these should be rewritten
	// within the header. E.g. "hostname.com:8080" -- though, common ports like
	// 80 and 443 are left out.

	// assign the origin host to the host header value, ONLY if it matches the domains
	// hostname
	curHost := strings.ToLower(req.URL.Host)
	origHost := strings.ToLower(c.Host)
	if curHost == origHost || curHost == "www."+origHost || "www."+curHost == origHost {
		req.Host = req.URL.Host

		// and overwrite the host used to make the connection
		if len(c.IP) > 0 {
			req.URL.Host = c.IP
		}
	}

	// update our cached resulting uri
	c.ResultURL = req.URL
	if len(req.Host) > 0 {
		c.ResultURL.Host = req.Host
	}

	return req
}

type HostnameError struct {
	Certificate *x509.Certificate
	Host        string
}

func (h HostnameError) Error() string {
	c := h.Certificate

	var valid string
	if ip := net.ParseIP(h.Host); ip != nil {
		// Trying to validate an IP
		if len(c.IPAddresses) == 0 {
			return "x509: cannot validate certificate for " + h.Host + " because it doesn't contain any IP SANs"
		}
		for _, san := range c.IPAddresses {
			if len(valid) > 0 {
				valid += ", "
			}
			valid += san.String()
		}
	} else {
		if len(c.DNSNames) > 0 {
			valid = strings.Join(c.DNSNames, ", ")
		} else {
			valid = c.Subject.CommonName
		}
	}
	return "x509: certificate is valid for " + valid + ", not " + h.Host
}

// toLowerCaseASCII returns a lower-case version of in. See RFC 6125 6.4.1. We use
// an explicitly ASCII function to avoid any sharp corners resulting from
// performing Unicode operations on DNS labels.
func toLowerCaseASCII(in string) string {
	// If the string is already lower-case then there's nothing to do.
	isAlreadyLowerCase := true
	for _, c := range in {
		if c == utf8.RuneError {
			// If we get a UTF-8 error then there might be
			// upper-case ASCII bytes in the invalid sequence.
			isAlreadyLowerCase = false
			break
		}
		if 'A' <= c && c <= 'Z' {
			isAlreadyLowerCase = false
			break
		}
	}

	if isAlreadyLowerCase {
		return in
	}

	out := []byte(in)
	for i, c := range out {
		if 'A' <= c && c <= 'Z' {
			out[i] += 'a' - 'A'
		}
	}
	return string(out)
}

func matchHostnames(pattern, host string) bool {
	host = strings.TrimSuffix(host, ".")
	pattern = strings.TrimSuffix(pattern, ".")

	if len(pattern) == 0 || len(host) == 0 {
		return false
	}

	patternParts := strings.Split(pattern, ".")
	hostParts := strings.Split(host, ".")

	if len(patternParts) != len(hostParts) {
		return false
	}

	for i, patternPart := range patternParts {
		if i == 0 && patternPart == "*" {
			continue
		}
		if patternPart != hostParts[i] {
			return false
		}
	}

	return true
}

// verifyx509 returns nil if c is a valid certificate for the named host.
// Otherwise it returns an error describing the mismatch.
func verifyx509(c *x509.Certificate, h string) error {
	// IP addresses may be written in [ ].
	candidateIP := h
	if len(h) >= 3 && h[0] == '[' && h[len(h)-1] == ']' {
		candidateIP = h[1 : len(h)-1]
	}
	if ip := net.ParseIP(candidateIP); ip != nil {
		// We only match IP addresses against IP SANs.
		// https://tools.ietf.org/html/rfc6125#appendix-B.2
		for _, candidate := range c.IPAddresses {
			if ip.Equal(candidate) {
				return nil
			}
		}
		return HostnameError{c, candidateIP}
	}

	lowered := toLowerCaseASCII(h)

	if len(c.DNSNames) > 0 {
		for _, match := range c.DNSNames {
			if matchHostnames(toLowerCaseASCII(match), lowered) {
				return nil
			}
		}
		// If Subject Alt Name is given, we ignore the common name.
	} else if matchHostnames(toLowerCaseASCII(c.Subject.CommonName), lowered) {
		return nil
	}

	return HostnameError{c, h}
}

func VerifyHostname(c *tls.ConnectionState, host string) error {
	if c == nil {
		return nil
	}

	// if len(c.VerifiedChains) == 0 {
	// 	return errors.New("tls: handshake did not verify certificate chain")
	// }

	return verifyx509(c.PeerCertificates[0], host)
}

// getHandler wraps the standard net/http library, allowing us to spoof hostnames and IP addresses
func (c *CustomClient) getHandler() (*CustomResponse, error) {
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{
			// unfortunately, ServerName will not persist over a redirect. so... we have to ignore
			// ssl invalidations and do them somewhat manually.
			InsecureSkipVerify: true,
			ServerName:         c.Host,
		},
	}
	client := &http.Client{
		CheckRedirect: c.redirectHandler,
		Timeout:       time.Duration(10) * time.Second,
		Transport:     transport,
	}

	req, err := http.NewRequest("GET", c.URL, nil)

	if err != nil {
		return nil, err
	}

	c.OriginURL = req.URL // set origin url for use in redirect wrapper
	c.requestWrap(req)

	// start tracking how long the request is going to take
	timer := NewTimer()

	// actually make the request here
	resp, err := client.Do(req)

	// stop tracking the request
	timer.End()

	var url string

	if err == nil {
		url = c.ResultURL.String()
		if err := VerifyHostname(resp.TLS, c.ResultURL.Host); err != nil {
			return nil, err
		}
	} else {
		url = req.URL.String()
	}

	wrappedResp := &CustomResponse{resp, timer.Result, url}

	return wrappedResp, err
}

// Get wraps GetHandler -- easy interface for making get requests
func Get(url string, ip string) (*CustomResponse, error) {
	host, err := getHost(url)
	if err != nil {
		return nil, err
	}

	c := &CustomClient{URL: url, IP: ip, Host: host}

	return c.getHandler()
}
<commit_msg>prevent some invalid ip addresses<commit_after>package scraper

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"
)

// CustomClient is the state for our custom http wrapper, which houses
// the needed data to be able to rewrite the outgoing request during
// redirects.
type CustomClient struct {
	URL       string
	IP        string
	Host      string
	ResultURL *url.URL // represents the url for the resulting request, without modifications
	OriginURL *url.URL // represents the url from the original request, without modifications
}

// CustomResponse is the wrapped response from http.Client.Do() which also
// includes a timer of how long the request took, and a few other minor
// extras.
type CustomResponse struct {
	*http.Response
	Time *TimerResult
	URL  string
}

var reIP = regexp.MustCompile(`^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$`)

func (c *CustomClient) redirectHandler(req *http.Request, via []*http.Request) error {
	c.requestWrap(req)

	// rewrite Referer (Referrer) if it exists, to have the proper hostname
	uri := via[len(via)-1].URL
	uri.Host = via[len(via)-1].Host
	req.Header.Set("Referer", uri.String())

	if len(via) > 3 {
		// assume too many redirects
		return errors.New("too many redirects (3)")
	}

	if reIP.MatchString(req.Host) && req.Host != c.IP {
		return errors.New("Redirected to IP that doesn't match proxy")
	}

	cHost := strings.ToLower(req.URL.Host)
	oHost := strings.ToLower(c.Host)
	if cHost != oHost && cHost != "www."+oHost && "www."+cHost != oHost {
		if c.OriginURL.Path == "" {
			return errors.New("Redirection does not match origin host")
		}
	}

	return nil
}

func (c *CustomClient) requestWrap(req *http.Request) *http.Request {
	// spoof useragent, as there are going to be sites/servers that are
	// setup to deny by a specific useragent string (or lack there of)
	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36")

	// if an IP address is provided, rewrite the Host headers
	// of note: if we plan to support custom ports, these should be rewritten
	// within the header. E.g. "hostname.com:8080" -- though, common ports like
	// 80 and 443 are left out.

	// assign the origin host to the host header value, ONLY if it matches the domains
	// hostname
	curHost := strings.ToLower(req.URL.Host)
	origHost := strings.ToLower(c.Host)
	if curHost == origHost || curHost == "www."+origHost || "www."+curHost == origHost {
		req.Host = req.URL.Host

		// and overwrite the host used to make the connection
		if len(c.IP) > 0 {
			req.URL.Host = c.IP
		}
	}

	// update our cached resulting uri
	c.ResultURL = req.URL
	if len(req.Host) > 0 {
		c.ResultURL.Host = req.Host
	}

	return req
}

type HostnameError struct {
	Certificate *x509.Certificate
	Host        string
}

func (h HostnameError) Error() string {
	c := h.Certificate

	var valid string
	if ip := net.ParseIP(h.Host); ip != nil {
		// Trying to validate an IP
		if len(c.IPAddresses) == 0 {
			return "x509: cannot validate certificate for " + h.Host + " because it doesn't contain any IP SANs"
		}
		for _, san := range c.IPAddresses {
			if len(valid) > 0 {
				valid += ", "
			}
			valid += san.String()
		}
	} else {
		if len(c.DNSNames) > 0 {
			valid = strings.Join(c.DNSNames, ", ")
		} else {
			valid = c.Subject.CommonName
		}
	}
	return "x509: certificate is valid for " + valid + ", not " + h.Host
}

// toLowerCaseASCII returns a lower-case version of in. See RFC 6125 6.4.1. We use
// an explicitly ASCII function to avoid any sharp corners resulting from
// performing Unicode operations on DNS labels.
func toLowerCaseASCII(in string) string {
	// If the string is already lower-case then there's nothing to do.
	isAlreadyLowerCase := true
	for _, c := range in {
		if c == utf8.RuneError {
			// If we get a UTF-8 error then there might be
			// upper-case ASCII bytes in the invalid sequence.
			isAlreadyLowerCase = false
			break
		}
		if 'A' <= c && c <= 'Z' {
			isAlreadyLowerCase = false
			break
		}
	}

	if isAlreadyLowerCase {
		return in
	}

	out := []byte(in)
	for i, c := range out {
		if 'A' <= c && c <= 'Z' {
			out[i] += 'a' - 'A'
		}
	}
	return string(out)
}

func matchHostnames(pattern, host string) bool {
	host = strings.TrimSuffix(host, ".")
	pattern = strings.TrimSuffix(pattern, ".")

	if len(pattern) == 0 || len(host) == 0 {
		return false
	}

	patternParts := strings.Split(pattern, ".")
	hostParts := strings.Split(host, ".")

	if len(patternParts) != len(hostParts) {
		return false
	}

	for i, patternPart := range patternParts {
		if i == 0 && patternPart == "*" {
			continue
		}
		if patternPart != hostParts[i] {
			return false
		}
	}

	return true
}

// verifyx509 returns nil if c is a valid certificate for the named host.
// Otherwise it returns an error describing the mismatch.
func verifyx509(c *x509.Certificate, h string) error {
	// IP addresses may be written in [ ].
	candidateIP := h
	if len(h) >= 3 && h[0] == '[' && h[len(h)-1] == ']' {
		candidateIP = h[1 : len(h)-1]
	}
	if ip := net.ParseIP(candidateIP); ip != nil {
		// We only match IP addresses against IP SANs.
		// https://tools.ietf.org/html/rfc6125#appendix-B.2
		for _, candidate := range c.IPAddresses {
			if ip.Equal(candidate) {
				return nil
			}
		}
		return HostnameError{c, candidateIP}
	}

	lowered := toLowerCaseASCII(h)

	if len(c.DNSNames) > 0 {
		for _, match := range c.DNSNames {
			if matchHostnames(toLowerCaseASCII(match), lowered) {
				return nil
			}
		}
		// If Subject Alt Name is given, we ignore the common name.
	} else if matchHostnames(toLowerCaseASCII(c.Subject.CommonName), lowered) {
		return nil
	}

	return HostnameError{c, h}
}

func VerifyHostname(c *tls.ConnectionState, host string) error {
	if c == nil {
		return nil
	}

	// if len(c.VerifiedChains) == 0 {
	// 	return errors.New("tls: handshake did not verify certificate chain")
	// }

	return verifyx509(c.PeerCertificates[0], host)
}

// getHandler wraps the standard net/http library, allowing us to spoof hostnames and IP addresses
func (c *CustomClient) getHandler() (*CustomResponse, error) {
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{
			// unfortunately, ServerName will not persist over a redirect. so... we have to ignore
			// ssl invalidations and do them somewhat manually.
			InsecureSkipVerify: true,
			ServerName:         c.Host,
		},
	}
	client := &http.Client{
		CheckRedirect: c.redirectHandler,
		Timeout:       time.Duration(10) * time.Second,
		Transport:     transport,
	}

	req, err := http.NewRequest("GET", c.URL, nil)

	if err != nil {
		return nil, err
	}

	c.OriginURL = req.URL // set origin url for use in redirect wrapper
	c.requestWrap(req)

	// start tracking how long the request is going to take
	timer := NewTimer()

	// actually make the request here
	resp, err := client.Do(req)

	// stop tracking the request
	timer.End()

	var url string

	if err == nil {
		url = c.ResultURL.String()
		if err := VerifyHostname(resp.TLS, c.ResultURL.Host); err != nil {
			return nil, err
		}
	} else {
		url = req.URL.String()
	}

	wrappedResp := &CustomResponse{resp, timer.Result, url}

	return wrappedResp, err
}

// Get wraps GetHandler -- easy interface for making get requests
func Get(url string, ip string) (*CustomResponse, error) {
	host, err := getHost(url)
	if err != nil {
		return nil, err
	}

	if len(ip) > 0 && !reIP.MatchString(ip) {
		return nil, errors.New("IP address provided is invalid")
	}

	c := &CustomClient{URL: url, IP: ip, Host: host}

	return c.getHandler()
}
<|endoftext|>


<commit_before>// Author: Liam Stanley <me@liamstanley.io>
// Docs: https://marill.liam.sh/
// Repo: https://github.com/Liamraystanley/marill

package scraper

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/Liamraystanley/marill/utils"
)

// CustomClient is the state for our custom http wrapper, which houses
// the needed data to be able to rewrite the outgoing request during
// redirects.
type CustomClient struct {
	URL       string
	Host      string
	ResultURL url.URL  // represents the url for the resulting request, without modifications
	OriginURL *url.URL // represents the url from the original request, without modifications
	ipmap     map[string]string
}

// CustomResponse is the wrapped response from http.Client.Do() which also
// includes a timer of how long the request took, and a few other minor
// extras.
type CustomResponse struct {
	*http.Response
	Time *utils.TimerResult
	URL  *url.URL
}

func (c *CustomClient) redirectHandler(req *http.Request, via []*http.Request) error {
	c.requestWrap(req)

	// rewrite Referer (Referrer) if it exists, to have the proper hostname
	uri := via[len(via)-1].URL
	uri.Host = via[len(via)-1].Host
	req.Header.Set("Referer", uri.String())

	if len(via) > 10 {
		// assume too many redirects
		return errors.New("too many redirects (10+)")
	}

	if reIP.MatchString(req.Host) && req.Host != c.Host {
		return errors.New("redirected to IP that doesn't match proxy/origin request")
	}

	// check to see if we're redirecting to a target which is possibly off this server
	// or not in this session of crawls
	if _, ok := c.ipmap[req.Host]; !ok {
		if c.OriginURL.Path == "" {
			// it's not in as a host -> ip map, but let's check to see if it resolves to a target ip
			var isin bool
			for _, val := range c.ipmap {
				if req.Host == val || req.URL.Host == val {
					isin = true
					break
				}
			}

			if !isin {
				return errors.New("redirection does not match origin host")
			}
		}
	}

	return nil
}

func (c *CustomClient) requestWrap(req *http.Request) *http.Request {
	// spoof useragent, as there are going to be sites/servers that are
	// setup to deny by a specific useragent string (or lack there of)
	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36")

	// if an IP address is provided, rewrite the Host headers
	// of note: if we plan to support custom ports, these should be rewritten
	// within the header. E.g. "hostname.com:8080" -- though, common ports like
	// 80 and 443 are left out.

	// assign the origin host to the host header value, ONLY if it matches the domains
	// hostname
	if ip, ok := c.ipmap[req.URL.Host]; ok {
		req.Host = req.URL.Host

		// and overwrite the host used to make the connection
		if len(ip) > 0 {
			req.URL.Host = ip
		}
	}

	// update our cached resulting uri
	c.ResultURL = *req.URL
	if len(req.Host) > 0 {
		c.ResultURL.Host = req.Host
	}

	return req
}

// HostnameError appears when an invalid SSL certificate is supplied
type HostnameError struct {
	Certificate *x509.Certificate
	Host        string
}

func (h HostnameError) Error() string {
	c := h.Certificate

	var valid string
	if ip := net.ParseIP(h.Host); ip != nil {
		// Trying to validate an IP
		if len(c.IPAddresses) == 0 {
			return "x509: cannot validate certificate for " + h.Host + " because it doesn't contain any IP SANs"
		}
		for _, san := range c.IPAddresses {
			if len(valid) > 0 {
				valid += ", "
			}
			valid += san.String()
		}
	} else {
		if len(c.DNSNames) > 0 {
			valid = strings.Join(c.DNSNames, ", ")
		} else {
			valid = c.Subject.CommonName
		}
	}
	return "x509: certificate is valid for " + valid + ", not " + h.Host
}

// toLowerCaseASCII returns a lower-case version of in. See RFC 6125 6.4.1. We use
// an explicitly ASCII function to avoid any sharp corners resulting from
// performing Unicode operations on DNS labels.
func toLowerCaseASCII(in string) string {
	// If the string is already lower-case then there's nothing to do.
	isAlreadyLowerCase := true
	for _, c := range in {
		if c == utf8.RuneError {
			// If we get a UTF-8 error then there might be
			// upper-case ASCII bytes in the invalid sequence.
			isAlreadyLowerCase = false
			break
		}
		if 'A' <= c && c <= 'Z' {
			isAlreadyLowerCase = false
			break
		}
	}

	if isAlreadyLowerCase {
		return in
	}

	out := []byte(in)
	for i, c := range out {
		if 'A' <= c && c <= 'Z' {
			out[i] += 'a' - 'A'
		}
	}
	return string(out)
}

func matchHostnames(pattern, host string) bool {
	host = strings.TrimSuffix(host, ".")
	pattern = strings.TrimSuffix(pattern, ".")

	if len(pattern) == 0 || len(host) == 0 {
		return false
	}

	patternParts := strings.Split(pattern, ".")
	hostParts := strings.Split(host, ".")

	if len(patternParts) != len(hostParts) {
		return false
	}

	for i, patternPart := range patternParts {
		if i == 0 && patternPart == "*" {
			continue
		}
		if patternPart != hostParts[i] {
			return false
		}
	}

	return true
}

// verifyx509 returns nil if c is a valid certificate for the named host.
// Otherwise it returns an error describing the mismatch.
func verifyx509(c *x509.Certificate, h string) error {
	// IP addresses may be written in [ ].
	candidateIP := h
	if len(h) >= 3 && h[0] == '[' && h[len(h)-1] == ']' {
		candidateIP = h[1 : len(h)-1]
	}
	if ip := net.ParseIP(candidateIP); ip != nil {
		// We only match IP addresses against IP SANs.
		// https://tools.ietf.org/html/rfc6125#appendix-B.2
		for _, candidate := range c.IPAddresses {
			if ip.Equal(candidate) {
				return nil
			}
		}
		return HostnameError{c, candidateIP}
	}

	lowered := toLowerCaseASCII(h)

	if len(c.DNSNames) > 0 {
		for _, match := range c.DNSNames {
			if matchHostnames(toLowerCaseASCII(match), lowered) {
				return nil
			}
		}
		// If Subject Alt Name is given, we ignore the common name.
	} else if matchHostnames(toLowerCaseASCII(c.Subject.CommonName), lowered) {
		return nil
	}

	return HostnameError{c, h}
}

// VerifyHostname verifies if the tls.ConnectionState certificate matches the hostname
func VerifyHostname(c *tls.ConnectionState, host string) error {
	if c == nil {
		return nil
	}

	return verifyx509(c.PeerCertificates[0], host)
}

// getHandler wraps the standard net/http library, allowing us to spoof hostnames and IP addresses
func (c *Crawler) getHandler(cl *CustomClient) (*CustomResponse, error) {
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{
			// unfortunately, ServerName will not persist over a redirect. so... we have to ignore
			// ssl invalidations and do them somewhat manually.
			InsecureSkipVerify: true,
			ServerName:         cl.Host,
		},
	}
	client := &http.Client{
		CheckRedirect: cl.redirectHandler,
		Timeout:       time.Duration(10) * time.Second,
		Transport:     transport,
	}

	req, err := http.NewRequest("GET", cl.URL, nil)

	if err != nil {
		return nil, err
	}

	cl.OriginURL = req.URL // set origin url for use in redirect wrapper
	cl.requestWrap(req)

	// start tracking how long the request is going to take
	timer := utils.NewTimer()

	// actually make the request here
	resp, err := client.Do(req)

	// stop tracking the request
	timer.End()

	if err == nil && !c.Cnf.AllowInsecure {
		if err = VerifyHostname(resp.TLS, cl.ResultURL.Host); err != nil {
			return nil, err
		}
	}

	if len(cl.ResultURL.Host) > 0 {
		return &CustomResponse{resp, timer.Result, &cl.ResultURL}, err
	}

	return &CustomResponse{resp, timer.Result, req.URL}, err
}

// Get wraps GetHandler -- easy interface for making get requests
func (c *Crawler) Get(url string) (*CustomResponse, error) {
	host, err := utils.GetHost(url)
	if err != nil {
		return nil, err
	}

	return c.getHandler(&CustomClient{URL: url, Host: host, ipmap: c.ipmap})
}
<commit_msg>add accept-language headers to request for sites that require this (closes #34)<commit_after>// Author: Liam Stanley <me@liamstanley.io>
// Docs: https://marill.liam.sh/
// Repo: https://github.com/Liamraystanley/marill

package scraper

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/Liamraystanley/marill/utils"
)

// CustomClient is the state for our custom http wrapper, which houses
// the needed data to be able to rewrite the outgoing request during
// redirects.
type CustomClient struct {
	URL       string
	Host      string
	ResultURL url.URL  // represents the url for the resulting request, without modifications
	OriginURL *url.URL // represents the url from the original request, without modifications
	ipmap     map[string]string
}

// CustomResponse is the wrapped response from http.Client.Do() which also
// includes a timer of how long the request took, and a few other minor
// extras.
type CustomResponse struct {
	*http.Response
	Time *utils.TimerResult
	URL  *url.URL
}

func (c *CustomClient) redirectHandler(req *http.Request, via []*http.Request) error {
	c.requestWrap(req)

	// add a few misc. headers here that are needed
	req.Header.Set("Accept-Language", "en-US,en;q=0.8")

	// rewrite Referer (Referrer) if it exists, to have the proper hostname
	uri := via[len(via)-1].URL
	uri.Host = via[len(via)-1].Host
	req.Header.Set("Referer", uri.String())

	if len(via) > 10 {
		// assume too many redirects
		return errors.New("too many redirects (10+)")
	}

	if reIP.MatchString(req.Host) && req.Host != c.Host {
		return errors.New("redirected to IP that doesn't match proxy/origin request")
	}

	// check to see if we're redirecting to a target which is possibly off this server
	// or not in this session of crawls
	if _, ok := c.ipmap[req.Host]; !ok {
		if c.OriginURL.Path == "" {
			// it's not in as a host -> ip map, but let's check to see if it resolves to a target ip
			var isin bool
			for _, val := range c.ipmap {
				if req.Host == val || req.URL.Host == val {
					isin = true
					break
				}
			}

			if !isin {
				return errors.New("redirection does not match origin host")
			}
		}
	}

	return nil
}

func (c *CustomClient) requestWrap(req *http.Request) *http.Request {
	// spoof useragent, as there are going to be sites/servers that are
	// setup to deny by a specific useragent string (or lack there of)
	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36")

	// if an IP address is provided, rewrite the Host headers
	// of note: if we plan to support custom ports, these should be rewritten
	// within the header. E.g. "hostname.com:8080" -- though, common ports like
	// 80 and 443 are left out.

	// assign the origin host to the host header value, ONLY if it matches the domains
	// hostname
	if ip, ok := c.ipmap[req.URL.Host]; ok {
		req.Host = req.URL.Host

		// and overwrite the host used to make the connection
		if len(ip) > 0 {
			req.URL.Host = ip
		}
	}

	// update our cached resulting uri
	c.ResultURL = *req.URL
	if len(req.Host) > 0 {
		c.ResultURL.Host = req.Host
	}

	return req
}

// HostnameError appears when an invalid SSL certificate is supplied
type HostnameError struct {
	Certificate *x509.Certificate
	Host        string
}

func (h HostnameError) Error() string {
	c := h.Certificate

	var valid string
	if ip := net.ParseIP(h.Host); ip != nil {
		// Trying to validate an IP
		if len(c.IPAddresses) == 0 {
			return "x509: cannot validate certificate for " + h.Host + " because it doesn't contain any IP SANs"
		}
		for _, san := range c.IPAddresses {
			if len(valid) > 0 {
				valid += ", "
			}
			valid += san.String()
		}
	} else {
		if len(c.DNSNames) > 0 {
			valid = strings.Join(c.DNSNames, ", ")
		} else {
			valid = c.Subject.CommonName
		}
	}
	return "x509: certificate is valid for " + valid + ", not " + h.Host
}

// toLowerCaseASCII returns a lower-case version of in. See RFC 6125 6.4.1. We use
// an explicitly ASCII function to avoid any sharp corners resulting from
// performing Unicode operations on DNS labels.
func toLowerCaseASCII(in string) string {
	// If the string is already lower-case then there's nothing to do.
	isAlreadyLowerCase := true
	for _, c := range in {
		if c == utf8.RuneError {
			// If we get a UTF-8 error then there might be
			// upper-case ASCII bytes in the invalid sequence.
			isAlreadyLowerCase = false
			break
		}
		if 'A' <= c && c <= 'Z' {
			isAlreadyLowerCase = false
			break
		}
	}

	if isAlreadyLowerCase {
		return in
	}

	out := []byte(in)
	for i, c := range out {
		if 'A' <= c && c <= 'Z' {
			out[i] += 'a' - 'A'
		}
	}
	return string(out)
}

func matchHostnames(pattern, host string) bool {
	host = strings.TrimSuffix(host, ".")
	pattern = strings.TrimSuffix(pattern, ".")

	if len(pattern) == 0 || len(host) == 0 {
		return false
	}

	patternParts := strings.Split(pattern, ".")
	hostParts := strings.Split(host, ".")

	if len(patternParts) != len(hostParts) {
		return false
	}

	for i, patternPart := range patternParts {
		if i == 0 && patternPart == "*" {
			continue
		}
		if patternPart != hostParts[i] {
			return false
		}
	}

	return true
}

// verifyx509 returns nil if c is a valid certificate for the named host.
// Otherwise it returns an error describing the mismatch.
func verifyx509(c *x509.Certificate, h string) error {
	// IP addresses may be written in [ ].
	candidateIP := h
	if len(h) >= 3 && h[0] == '[' && h[len(h)-1] == ']' {
		candidateIP = h[1 : len(h)-1]
	}
	if ip := net.ParseIP(candidateIP); ip != nil {
		// We only match IP addresses against IP SANs.
		// https://tools.ietf.org/html/rfc6125#appendix-B.2
		for _, candidate := range c.IPAddresses {
			if ip.Equal(candidate) {
				return nil
			}
		}
		return HostnameError{c, candidateIP}
	}

	lowered := toLowerCaseASCII(h)

	if len(c.DNSNames) > 0 {
		for _, match := range c.DNSNames {
			if matchHostnames(toLowerCaseASCII(match), lowered) {
				return nil
			}
		}
		// If Subject Alt Name is given, we ignore the common name.
	} else if matchHostnames(toLowerCaseASCII(c.Subject.CommonName), lowered) {
		return nil
	}

	return HostnameError{c, h}
}

// VerifyHostname verifies if the tls.ConnectionState certificate matches the hostname
func VerifyHostname(c *tls.ConnectionState, host string) error {
	if c == nil {
		return nil
	}

	return verifyx509(c.PeerCertificates[0], host)
}

// getHandler wraps the standard net/http library, allowing us to spoof hostnames and IP addresses
func (c *Crawler) getHandler(cl *CustomClient) (*CustomResponse, error) {
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{
			// unfortunately, ServerName will not persist over a redirect. so... we have to ignore
			// ssl invalidations and do them somewhat manually.
			InsecureSkipVerify: true,
			ServerName:         cl.Host,
		},
	}
	client := &http.Client{
		CheckRedirect: cl.redirectHandler,
		Timeout:       time.Duration(10) * time.Second,
		Transport:     transport,
	}

	req, err := http.NewRequest("GET", cl.URL, nil)

	if err != nil {
		return nil, err
	}

	cl.OriginURL = req.URL // set origin url for use in redirect wrapper
	cl.requestWrap(req)

	// start tracking how long the request is going to take
	timer := utils.NewTimer()

	// actually make the request here
	resp, err := client.Do(req)

	// stop tracking the request
	timer.End()

	if err == nil && !c.Cnf.AllowInsecure {
		if err = VerifyHostname(resp.TLS, cl.ResultURL.Host); err != nil {
			return nil, err
		}
	}

	if len(cl.ResultURL.Host) > 0 {
		return &CustomResponse{resp, timer.Result, &cl.ResultURL}, err
	}

	return &CustomResponse{resp, timer.Result, req.URL}, err
}

// Get wraps GetHandler -- easy interface for making get requests
func (c *Crawler) Get(url string) (*CustomResponse, error) {
	host, err := utils.GetHost(url)
	if err != nil {
		return nil, err
	}

	return c.getHandler(&CustomClient{URL: url, Host: host, ipmap: c.ipmap})
}
<|endoftext|>


<commit_before>// Copyright 2015 Davis Webb
// Copyright 2015 Zhandos Suleimenov
// Copyright 2015 Luke Shumaker

package domain_handlers

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/mail"
	"net/url"
	"periwinkle"
	"periwinkle/backend"
	"periwinkle/twilio"
	"postfixpipe"
	"strings"
	//"os"
)

func HandleSMS(r io.Reader, name string, db *periwinkle.Tx, cfg *periwinkle.Cfg) postfixpipe.ExitStatus {
	message, err := mail.ReadMessage(r)
	if err != nil {
		log.Println(err)
		return postfixpipe.EX_NOINPUT
	}
	status, err := sender(*message, name, db, cfg)
	if err != nil {
		log.Println(err)
		return postfixpipe.EX_NOINPUT
	}
	log.Println(status)
	return postfixpipe.EX_OK
}

// Returns the status of the message: queued, sending, sent,
// delivered, undelivered, failed.  If an error occurs, it returns
// Error.
func sender(message mail.Message, smsTo string, db *periwinkle.Tx, cfg *periwinkle.Cfg) (status string, err error) {

	group := message.Header.Get("From")
	user := backend.GetUserByAddress(db, "sms", smsTo)

	smsFrom := backend.GetTwilioNumberByUserAndGroup(db, user.ID, strings.Split(group, "@")[0])

	if smsFrom == "" {

		twilio_num := twilio.GetUnusedTwilioNumbersByUser(cfg, db, user.ID)
		if twilio_num == nil {
			new_num, err := twilio.NewPhoneNum(cfg)
			if err != nil {
				return "", err
			}
			backend.AssignTwilioNumber(db, user.ID, strings.Split(group, "@")[0], new_num)
			smsFrom = new_num
		} else {
			backend.AssignTwilioNumber(db, user.ID, strings.Split(group, "@")[0], twilio_num[0])
			smsFrom = twilio_num[0]
		}
	}

	smsBody := message.Header.Get("Subject")
	//smsBody, err := ioutil.ReadAll(message.Body)
	//if err != nil {
	//	return "", err
	//}

	messagesURL := "https://api.twilio.com/2010-04-01/Accounts/" + cfg.TwilioAccountID + "/Messages.json"

	v := url.Values{}
	v.Set("From", smsFrom)
	v.Set("To", smsTo)
	v.Set("Body", string(smsBody))
	v.Set("StatusCallback", cfg.WebRoot+"/callbacks/twilio-sms")
	//host,_ := os.Hostname()
	//v.Set("StatusCallback", "http://" + host + ":8080/callbacks/twilio-sms")
	client := &http.Client{}

	req, err := http.NewRequest("POST", messagesURL, bytes.NewBuffer([]byte(v.Encode())))
	if err != nil {
		return
	}
	req.SetBasicAuth(cfg.TwilioAccountID, cfg.TwilioAuthToken)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	resp, err := client.Do(req)
	defer resp.Body.Close()
	if err != nil {
		return "", err
	}

	if resp.StatusCode == 200 || resp.StatusCode == 201 {

		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return "", err
		}

		message := twilio.Message{}
		json.Unmarshal([]byte(body), &message)
		log.Println("106")
		smsStatus, err := SmsWaitForCallback(cfg, message.Sid)
		log.Println("108")
		if err != nil {
			return "", err
		}

		if smsStatus.MessageStatus == "undelivered" || smsStatus.MessageStatus == "failed" {
			return smsStatus.MessageStatus, fmt.Errorf("%s", smsStatus.ErrorCode)
		}
		log.Println("116")
		if smsStatus.MessageStatus == "queued" || smsStatus.MessageStatus == "sending" || smsStatus.MessageStatus == "sent" {
			smsStatus, err = SmsWaitForCallback(cfg, message.Sid)

			if err != nil {
				return "", err
			}
		}

		if smsStatus.MessageStatus == "undelivered" || smsStatus.MessageStatus == "failed" {
			return smsStatus.MessageStatus, fmt.Errorf("%s", smsStatus.ErrorCode)
		}

		status = smsStatus.MessageStatus
		err = nil
		return status, err
	} else {
		err = fmt.Errorf("%s", resp.Status)
		return
	}
}
<commit_msg>anyrth<commit_after>// Copyright 2015 Davis Webb
// Copyright 2015 Zhandos Suleimenov
// Copyright 2015 Luke Shumaker

package domain_handlers

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/mail"
	"net/url"
	"periwinkle"
	"periwinkle/backend"
	"periwinkle/twilio"
	"postfixpipe"
	"strings"
	//"os"
)

func HandleSMS(r io.Reader, name string, db *periwinkle.Tx, cfg *periwinkle.Cfg) postfixpipe.ExitStatus {
	message, err := mail.ReadMessage(r)
	if err != nil {
		log.Println(err)
		return postfixpipe.EX_NOINPUT
	}
	status, err := sender(*message, name, db, cfg)
	if err != nil {
		log.Println(err)
		return postfixpipe.EX_NOINPUT
	}
	log.Println(status)
	return postfixpipe.EX_OK
}

// Returns the status of the message: queued, sending, sent,
// delivered, undelivered, failed.  If an error occurs, it returns
// Error.
func sender(message mail.Message, smsTo string, db *periwinkle.Tx, cfg *periwinkle.Cfg) (status string, err error) {

	group := message.Header.Get("From")
	user := backend.GetUserByAddress(db, "sms", smsTo)

	smsFrom := backend.GetTwilioNumberByUserAndGroup(db, user.ID, strings.Split(group, "@")[0])

	if smsFrom == "" {

		twilio_num := twilio.GetUnusedTwilioNumbersByUser(cfg, db, user.ID)
		if twilio_num == nil {
			new_num, err := twilio.NewPhoneNum(cfg)
			if err != nil {
				return "", err
			}
			backend.AssignTwilioNumber(db, user.ID, strings.Split(group, "@")[0], new_num)
			smsFrom = new_num
		} else {
			backend.AssignTwilioNumber(db, user.ID, strings.Split(group, "@")[0], twilio_num[0])
			smsFrom = twilio_num[0]
		}
	}

	smsBody := message.Header.Get("Subject")
	//smsBody, err := ioutil.ReadAll(message.Body)
	//if err != nil {
	//	return "", err
	//}

	messagesURL := "https://api.twilio.com/2010-04-01/Accounts/" + cfg.TwilioAccountID + "/Messages.json"

	v := url.Values{}
	v.Set("From", smsFrom)
	v.Set("To", smsTo)
	v.Set("Body", string(smsBody))
	v.Set("StatusCallback", "http://" + cfg.WebRoot+":8080/callbacks/twilio-sms")
	//host,_ := os.Hostname()
	//v.Set("StatusCallback", "http://" + host + ":8080/callbacks/twilio-sms")
	client := &http.Client{}

	req, err := http.NewRequest("POST", messagesURL, bytes.NewBuffer([]byte(v.Encode())))
	if err != nil {
		return
	}
	req.SetBasicAuth(cfg.TwilioAccountID, cfg.TwilioAuthToken)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	resp, err := client.Do(req)
	defer resp.Body.Close()
	if err != nil {
		return "", err
	}

	if resp.StatusCode == 200 || resp.StatusCode == 201 {

		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return "", err
		}

		message := twilio.Message{}
		json.Unmarshal([]byte(body), &message)
		log.Println("106")
		smsStatus, err := SmsWaitForCallback(cfg, message.Sid)
		log.Println("108")
		if err != nil {
			return "", err
		}

		if smsStatus.MessageStatus == "undelivered" || smsStatus.MessageStatus == "failed" {
			return smsStatus.MessageStatus, fmt.Errorf("%s", smsStatus.ErrorCode)
		}
		log.Println("116")
		if smsStatus.MessageStatus == "queued" || smsStatus.MessageStatus == "sending" || smsStatus.MessageStatus == "sent" {
			smsStatus, err = SmsWaitForCallback(cfg, message.Sid)

			if err != nil {
				return "", err
			}
		}

		if smsStatus.MessageStatus == "undelivered" || smsStatus.MessageStatus == "failed" {
			return smsStatus.MessageStatus, fmt.Errorf("%s", smsStatus.ErrorCode)
		}

		status = smsStatus.MessageStatus
		err = nil
		return status, err
	} else {
		err = fmt.Errorf("%s", resp.Status)
		return
	}
}
<|endoftext|>