<commit_before>package libvirt

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
)

// network transparent image
type image interface {
	Size() (uint64, error)
	Import(func(io.Reader) error, defVolume) error
	String() string
}

type localImage struct {
	path string
}

func (i *localImage) String() string {
	return i.path
}

func (i *localImage) Size() (uint64, error) {
	file, err := os.Open(i.path)
	if err != nil {
		return 0, err
	}

	fi, err := file.Stat()
	if err != nil {
		return 0, err
	}
	return uint64(fi.Size()), nil
}

func (i *localImage) Import(copier func(io.Reader) error, vol defVolume) error {

	file, err := os.Open(i.path)
	defer file.Close()
	if err != nil {
		return fmt.Errorf("Error while opening %s: %s", i.path, err)
	}

	if fi, err := file.Stat(); err != nil {
		return err
	} else {
		// we can skip the upload if the modification times are the same
		if vol.Target.Timestamps != nil && vol.Target.Timestamps.Modification != nil {
			modTime := UnixTimestamp{fi.ModTime()}
			if modTime == *vol.Target.Timestamps.Modification {
				log.Printf("Modification time is the same: skipping image copy")
				return nil
			}
		}
	}

	return copier(file)
}

type httpImage struct {
	url *url.URL
}

func (i *httpImage) String() string {
	return i.url.String()
}

func (i *httpImage) Size() (uint64, error) {
	response, err := http.Head(i.url.String())
	if err != nil {
		return 0, err
	}
	length, err := strconv.Atoi(response.Header.Get("Content-Length"))
	if err != nil {
		return 0, err
	}
	return uint64(length), nil
}

func (i *httpImage) Import(copier func(io.Reader) error, vol defVolume) error {
	client := &http.Client{}
	req, _ := http.NewRequest("GET", i.url.String(), nil)

	if vol.Target.Timestamps != nil && vol.Target.Timestamps.Modification != nil {
		t := vol.Target.Timestamps.Modification.UTC().Format(http.TimeFormat)
		req.Header.Set("If-Modified-Since", t)
	}
	response, err := client.Do(req)
	defer response.Body.Close()

	if err != nil {
		return fmt.Errorf("Error while downloading %s: %s", i.url.String(), err)
	}
	if response.StatusCode == http.StatusNotModified {
		return nil
	}

	return copier(response.Body)
}

func newImage(source string) (image, error) {
	url, err := url.Parse(source)
	if err != nil {
		return nil, fmt.Errorf("Can't parse source '%s' as url: %s", source, err)
	}

	if strings.HasPrefix(url.Scheme, "http") {
		return &httpImage{url: url}, nil
	} else if url.Scheme == "file" || url.Scheme == "" {
		return &localImage{path: url.Path}, nil
	} else {
		return nil, fmt.Errorf("Don't know how to read from '%s': %s", url.String(), err)
	}
}
<commit_msg>Replace cryptic error with better one<commit_after>package libvirt

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
)

// network transparent image
type image interface {
	Size() (uint64, error)
	Import(func(io.Reader) error, defVolume) error
	String() string
}

type localImage struct {
	path string
}

func (i *localImage) String() string {
	return i.path
}

func (i *localImage) Size() (uint64, error) {
	file, err := os.Open(i.path)
	if err != nil {
		return 0, err
	}

	fi, err := file.Stat()
	if err != nil {
		return 0, err
	}
	return uint64(fi.Size()), nil
}

func (i *localImage) Import(copier func(io.Reader) error, vol defVolume) error {

	file, err := os.Open(i.path)
	defer file.Close()
	if err != nil {
		return fmt.Errorf("Error while opening %s: %s", i.path, err)
	}

	if fi, err := file.Stat(); err != nil {
		return err
	} else {
		// we can skip the upload if the modification times are the same
		if vol.Target.Timestamps != nil && vol.Target.Timestamps.Modification != nil {
			modTime := UnixTimestamp{fi.ModTime()}
			if modTime == *vol.Target.Timestamps.Modification {
				log.Printf("Modification time is the same: skipping image copy")
				return nil
			}
		}
	}

	return copier(file)
}

type httpImage struct {
	url *url.URL
}

func (i *httpImage) String() string {
	return i.url.String()
}

func (i *httpImage) Size() (uint64, error) {
	response, err := http.Head(i.url.String())
	if err != nil {
		return 0, err
	}
	if response.StatusCode != 200 {
		return 0,
			fmt.Errorf(
				"Error accessing remote resource: %s - %s",
				i.url.String(),
				response.Status)
	}

	length, err := strconv.Atoi(response.Header.Get("Content-Length"))
	if err != nil {
		err = fmt.Errorf(
			"Error while getting Content-Length of \"%s\": %s - got %s",
			i.url.String(),
			err,
			response.Header.Get("Content-Length"))
		return 0, err
	}
	return uint64(length), nil
}

func (i *httpImage) Import(copier func(io.Reader) error, vol defVolume) error {
	client := &http.Client{}
	req, _ := http.NewRequest("GET", i.url.String(), nil)

	if vol.Target.Timestamps != nil && vol.Target.Timestamps.Modification != nil {
		t := vol.Target.Timestamps.Modification.UTC().Format(http.TimeFormat)
		req.Header.Set("If-Modified-Since", t)
	}
	response, err := client.Do(req)
	defer response.Body.Close()

	if err != nil {
		return fmt.Errorf("Error while downloading %s: %s", i.url.String(), err)
	}
	if response.StatusCode == http.StatusNotModified {
		return nil
	}

	return copier(response.Body)
}

func newImage(source string) (image, error) {
	url, err := url.Parse(source)
	if err != nil {
		return nil, fmt.Errorf("Can't parse source '%s' as url: %s", source, err)
	}

	if strings.HasPrefix(url.Scheme, "http") {
		return &httpImage{url: url}, nil
	} else if url.Scheme == "file" || url.Scheme == "" {
		return &localImage{path: url.Path}, nil
	} else {
		return nil, fmt.Errorf("Don't know how to read from '%s': %s", url.String(), err)
	}
}
<|endoftext|>


<commit_before>// Package bundle manages translations for multiple languages.
package bundle

import (
	"encoding/json"
	"fmt"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"reflect"

	"path/filepath"

	"github.com/nicksnyder/go-i18n/i18n/language"
	"github.com/nicksnyder/go-i18n/i18n/translation"
)

// TranslateFunc is a copy of i18n.TranslateFunc to avoid a circular dependency.
type TranslateFunc func(translationID string, args ...interface{}) string

// Bundle stores the translations for multiple languages.
type Bundle struct {
	// The primary translations for a language tag and translation id.
	translations map[string]map[string]translation.Translation

	// Translations that can be used when an exact language match is not possible.
	fallbackTranslations map[string]map[string]translation.Translation
}

// New returns an empty bundle.
func New() *Bundle {
	return &Bundle{
		translations:         make(map[string]map[string]translation.Translation),
		fallbackTranslations: make(map[string]map[string]translation.Translation),
	}
}

// MustLoadTranslationFile is similar to LoadTranslationFile
// except it panics if an error happens.
func (b *Bundle) MustLoadTranslationFile(filename string) {
	if err := b.LoadTranslationFile(filename); err != nil {
		panic(err)
	}
}

// LoadTranslationFile loads the translations from filename into memory.
//
// The language that the translations are associated with is parsed from the filename (e.g. en-US.json).
//
// Generally you should load translation files once during your program's initialization.
func (b *Bundle) LoadTranslationFile(filename string) error {
	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}
	return b.ParseTranslationFileBytes(filename, buf)
}

// ParseTranslationFileBytes is similar to LoadTranslationFile except it parses the bytes in buf.
//
// It is useful for parsing translation files embedded with go-bindata.
func (b *Bundle) ParseTranslationFileBytes(filename string, buf []byte) error {
	basename := filepath.Base(filename)
	langs := language.Parse(basename)
	switch l := len(langs); {
	case l == 0:
		return fmt.Errorf("no language found in %q", basename)
	case l > 1:
		return fmt.Errorf("multiple languages found in filename %q: %v; expected one", basename, langs)
	}
	translations, err := parseTranslations(filename, buf)
	if err != nil {
		return err
	}
	b.AddTranslation(langs[0], translations...)
	return nil
}

func parseTranslations(filename string, buf []byte) ([]translation.Translation, error) {
	var unmarshalFunc func([]byte, interface{}) error
	switch format := filepath.Ext(filename); format {
	case ".json":
		unmarshalFunc = json.Unmarshal
	case ".yaml":
		unmarshalFunc = yaml.Unmarshal
	default:
		return nil, fmt.Errorf("unsupported file extension %s", format)
	}

	var translationsData []map[string]interface{}
	if len(buf) > 0 {
		if err := unmarshalFunc(buf, &translationsData); err != nil {
			return nil, err
		}
	}

	translations := make([]translation.Translation, 0, len(translationsData))
	for i, translationData := range translationsData {
		t, err := translation.NewTranslation(translationData)
		if err != nil {
			return nil, fmt.Errorf("unable to parse translation #%d in %s because %s\n%v", i, filename, err, translationData)
		}
		translations = append(translations, t)
	}
	return translations, nil
}

// AddTranslation adds translations for a language.
//
// It is useful if your translations are in a format not supported by LoadTranslationFile.
func (b *Bundle) AddTranslation(lang *language.Language, translations ...translation.Translation) {
	if b.translations[lang.Tag] == nil {
		b.translations[lang.Tag] = make(map[string]translation.Translation, len(translations))
	}
	currentTranslations := b.translations[lang.Tag]
	for _, newTranslation := range translations {
		if currentTranslation := currentTranslations[newTranslation.ID()]; currentTranslation != nil {
			currentTranslations[newTranslation.ID()] = currentTranslation.Merge(newTranslation)
		} else {
			currentTranslations[newTranslation.ID()] = newTranslation
		}
	}

	// lang can provide translations for less specific language tags.
	for _, tag := range lang.MatchingTags() {
		b.fallbackTranslations[tag] = currentTranslations
	}
}

// Translations returns all translations in the bundle.
func (b *Bundle) Translations() map[string]map[string]translation.Translation {
	return b.translations
}

// LanguageTags returns the tags of all languages that that have been added.
func (b *Bundle) LanguageTags() []string {
	var tags []string
	for k := range b.translations {
		tags = append(tags, k)
	}
	return tags
}

// LanguageTranslationIDs returns the ids of all translations that have been added for a given language.
func (b *Bundle) LanguageTranslationIDs(languageTag string) []string {
	var ids []string
	for id := range b.translations[languageTag] {
		ids = append(ids, id)
	}
	return ids
}

// MustTfunc is similar to Tfunc except it panics if an error happens.
func (b *Bundle) MustTfunc(pref string, prefs ...string) TranslateFunc {
	tfunc, err := b.Tfunc(pref, prefs...)
	if err != nil {
		panic(err)
	}
	return tfunc
}

// MustTfuncAndLanguage is similar to TfuncAndLanguage except it panics if an error happens.
func (b *Bundle) MustTfuncAndLanguage(pref string, prefs ...string) (TranslateFunc, *language.Language) {
	tfunc, language, err := b.TfuncAndLanguage(pref, prefs...)
	if err != nil {
		panic(err)
	}
	return tfunc, language
}

// Tfunc is similar to TfuncAndLanguage except is doesn't return the Language.
func (b *Bundle) Tfunc(pref string, prefs ...string) (TranslateFunc, error) {
	tfunc, _, err := b.TfuncAndLanguage(pref, prefs...)
	return tfunc, err
}

// TfuncAndLanguage returns a TranslateFunc for the first Language that
// has a non-zero number of translations in the bundle.
//
// The returned Language matches the the first language preference that could be satisfied,
// but this may not strictly match the language of the translations used to satisfy that preference.
//
// For example, the user may request "zh". If there are no translations for "zh" but there are translations
// for "zh-cn", then the translations for "zh-cn" will be used but the returned Language will be "zh".
//
// It can parse languages from Accept-Language headers (RFC 2616),
// but it assumes weights are monotonically decreasing.
func (b *Bundle) TfuncAndLanguage(pref string, prefs ...string) (TranslateFunc, *language.Language, error) {
	lang := b.supportedLanguage(pref, prefs...)
	var err error
	if lang == nil {
		err = fmt.Errorf("no supported languages found %#v", append(prefs, pref))
	}
	return func(translationID string, args ...interface{}) string {
		return b.translate(lang, translationID, args...)
	}, lang, err
}

// supportedLanguage returns the first language which
// has a non-zero number of translations in the bundle.
func (b *Bundle) supportedLanguage(pref string, prefs ...string) *language.Language {
	lang := b.translatedLanguage(pref)
	if lang == nil {
		for _, pref := range prefs {
			lang = b.translatedLanguage(pref)
			if lang != nil {
				break
			}
		}
	}
	return lang
}

func (b *Bundle) translatedLanguage(src string) *language.Language {
	langs := language.Parse(src)
	for _, lang := range langs {
		if len(b.translations[lang.Tag]) > 0 ||
			len(b.fallbackTranslations[lang.Tag]) > 0 {
			return lang
		}
	}
	return nil
}

func (b *Bundle) translate(lang *language.Language, translationID string, args ...interface{}) string {
	if lang == nil {
		return translationID
	}

	translations := b.translations[lang.Tag]
	if translations == nil {
		translations = b.fallbackTranslations[lang.Tag]
		if translations == nil {
			return translationID
		}
	}

	translation := translations[translationID]
	if translation == nil {
		return translationID
	}

	var data interface{}
	var count interface{}
	if argc := len(args); argc > 0 {
		if isNumber(args[0]) {
			count = args[0]
			if argc > 1 {
				data = args[1]
			}
		} else {
			data = args[0]
		}
	}

	if count != nil {
		if data == nil {
			data = map[string]interface{}{"Count": count}
		} else {
			dataMap := toMap(data)
			dataMap["Count"] = count
			data = dataMap
		}
	} else {
		dataMap := toMap(data)
		if c, ok := dataMap["Count"]; ok {
			count = c
		}
	}

	p, _ := lang.Plural(count)
	template := translation.Template(p)
	if template == nil {
		return translationID
	}

	s := template.Execute(data)
	if s == "" {
		return translationID
	}
	return s
}

func isNumber(n interface{}) bool {
	switch n.(type) {
	case int, int8, int16, int32, int64, string:
		return true
	}
	return false
}

func toMap(input interface{}) map[string]interface{} {
	if data, ok := input.(map[string]interface{}); ok {
		return data
	}
	v := reflect.ValueOf(input)
	switch v.Kind() {
	case reflect.Ptr:
		return toMap(v.Elem().Interface())
	case reflect.Struct:
		return structToMap(v)
	default:
		return nil
	}
}

// Converts the top level of a struct to a map[string]interface{}.
// Code inspired by github.com/fatih/structs.
func structToMap(v reflect.Value) map[string]interface{} {
	out := make(map[string]interface{})
	t := v.Type()
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		if field.PkgPath != "" {
			// unexported field. skip.
			continue
		}
		out[field.Name] = v.FieldByName(field.Name).Interface()
	}
	return out
}
<commit_msg>report errored filename when it failed to parse<commit_after>// Package bundle manages translations for multiple languages.
package bundle

import (
	"encoding/json"
	"fmt"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"reflect"

	"path/filepath"

	"github.com/nicksnyder/go-i18n/i18n/language"
	"github.com/nicksnyder/go-i18n/i18n/translation"
)

// TranslateFunc is a copy of i18n.TranslateFunc to avoid a circular dependency.
type TranslateFunc func(translationID string, args ...interface{}) string

// Bundle stores the translations for multiple languages.
type Bundle struct {
	// The primary translations for a language tag and translation id.
	translations map[string]map[string]translation.Translation

	// Translations that can be used when an exact language match is not possible.
	fallbackTranslations map[string]map[string]translation.Translation
}

// New returns an empty bundle.
func New() *Bundle {
	return &Bundle{
		translations:         make(map[string]map[string]translation.Translation),
		fallbackTranslations: make(map[string]map[string]translation.Translation),
	}
}

// MustLoadTranslationFile is similar to LoadTranslationFile
// except it panics if an error happens.
func (b *Bundle) MustLoadTranslationFile(filename string) {
	if err := b.LoadTranslationFile(filename); err != nil {
		panic(err)
	}
}

// LoadTranslationFile loads the translations from filename into memory.
//
// The language that the translations are associated with is parsed from the filename (e.g. en-US.json).
//
// Generally you should load translation files once during your program's initialization.
func (b *Bundle) LoadTranslationFile(filename string) error {
	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}
	return b.ParseTranslationFileBytes(filename, buf)
}

// ParseTranslationFileBytes is similar to LoadTranslationFile except it parses the bytes in buf.
//
// It is useful for parsing translation files embedded with go-bindata.
func (b *Bundle) ParseTranslationFileBytes(filename string, buf []byte) error {
	basename := filepath.Base(filename)
	langs := language.Parse(basename)
	switch l := len(langs); {
	case l == 0:
		return fmt.Errorf("no language found in %q", basename)
	case l > 1:
		return fmt.Errorf("multiple languages found in filename %q: %v; expected one", basename, langs)
	}
	translations, err := parseTranslations(filename, buf)
	if err != nil {
		return err
	}
	b.AddTranslation(langs[0], translations...)
	return nil
}

func parseTranslations(filename string, buf []byte) ([]translation.Translation, error) {
	var unmarshalFunc func([]byte, interface{}) error
	switch format := filepath.Ext(filename); format {
	case ".json":
		unmarshalFunc = json.Unmarshal
	case ".yaml":
		unmarshalFunc = yaml.Unmarshal
	default:
		return nil, fmt.Errorf("unsupported file extension %s", format)
	}

	var translationsData []map[string]interface{}
	if len(buf) > 0 {
		if err := unmarshalFunc(buf, &translationsData); err != nil {
			return nil, fmt.Errorf("failed to load %s because %s", filename, err)
		}
	}

	translations := make([]translation.Translation, 0, len(translationsData))
	for i, translationData := range translationsData {
		t, err := translation.NewTranslation(translationData)
		if err != nil {
			return nil, fmt.Errorf("unable to parse translation #%d in %s because %s\n%v", i, filename, err, translationData)
		}
		translations = append(translations, t)
	}
	return translations, nil
}

// AddTranslation adds translations for a language.
//
// It is useful if your translations are in a format not supported by LoadTranslationFile.
func (b *Bundle) AddTranslation(lang *language.Language, translations ...translation.Translation) {
	if b.translations[lang.Tag] == nil {
		b.translations[lang.Tag] = make(map[string]translation.Translation, len(translations))
	}
	currentTranslations := b.translations[lang.Tag]
	for _, newTranslation := range translations {
		if currentTranslation := currentTranslations[newTranslation.ID()]; currentTranslation != nil {
			currentTranslations[newTranslation.ID()] = currentTranslation.Merge(newTranslation)
		} else {
			currentTranslations[newTranslation.ID()] = newTranslation
		}
	}

	// lang can provide translations for less specific language tags.
	for _, tag := range lang.MatchingTags() {
		b.fallbackTranslations[tag] = currentTranslations
	}
}

// Translations returns all translations in the bundle.
func (b *Bundle) Translations() map[string]map[string]translation.Translation {
	return b.translations
}

// LanguageTags returns the tags of all languages that that have been added.
func (b *Bundle) LanguageTags() []string {
	var tags []string
	for k := range b.translations {
		tags = append(tags, k)
	}
	return tags
}

// LanguageTranslationIDs returns the ids of all translations that have been added for a given language.
func (b *Bundle) LanguageTranslationIDs(languageTag string) []string {
	var ids []string
	for id := range b.translations[languageTag] {
		ids = append(ids, id)
	}
	return ids
}

// MustTfunc is similar to Tfunc except it panics if an error happens.
func (b *Bundle) MustTfunc(pref string, prefs ...string) TranslateFunc {
	tfunc, err := b.Tfunc(pref, prefs...)
	if err != nil {
		panic(err)
	}
	return tfunc
}

// MustTfuncAndLanguage is similar to TfuncAndLanguage except it panics if an error happens.
func (b *Bundle) MustTfuncAndLanguage(pref string, prefs ...string) (TranslateFunc, *language.Language) {
	tfunc, language, err := b.TfuncAndLanguage(pref, prefs...)
	if err != nil {
		panic(err)
	}
	return tfunc, language
}

// Tfunc is similar to TfuncAndLanguage except is doesn't return the Language.
func (b *Bundle) Tfunc(pref string, prefs ...string) (TranslateFunc, error) {
	tfunc, _, err := b.TfuncAndLanguage(pref, prefs...)
	return tfunc, err
}

// TfuncAndLanguage returns a TranslateFunc for the first Language that
// has a non-zero number of translations in the bundle.
//
// The returned Language matches the the first language preference that could be satisfied,
// but this may not strictly match the language of the translations used to satisfy that preference.
//
// For example, the user may request "zh". If there are no translations for "zh" but there are translations
// for "zh-cn", then the translations for "zh-cn" will be used but the returned Language will be "zh".
//
// It can parse languages from Accept-Language headers (RFC 2616),
// but it assumes weights are monotonically decreasing.
func (b *Bundle) TfuncAndLanguage(pref string, prefs ...string) (TranslateFunc, *language.Language, error) {
	lang := b.supportedLanguage(pref, prefs...)
	var err error
	if lang == nil {
		err = fmt.Errorf("no supported languages found %#v", append(prefs, pref))
	}
	return func(translationID string, args ...interface{}) string {
		return b.translate(lang, translationID, args...)
	}, lang, err
}

// supportedLanguage returns the first language which
// has a non-zero number of translations in the bundle.
func (b *Bundle) supportedLanguage(pref string, prefs ...string) *language.Language {
	lang := b.translatedLanguage(pref)
	if lang == nil {
		for _, pref := range prefs {
			lang = b.translatedLanguage(pref)
			if lang != nil {
				break
			}
		}
	}
	return lang
}

func (b *Bundle) translatedLanguage(src string) *language.Language {
	langs := language.Parse(src)
	for _, lang := range langs {
		if len(b.translations[lang.Tag]) > 0 ||
			len(b.fallbackTranslations[lang.Tag]) > 0 {
			return lang
		}
	}
	return nil
}

func (b *Bundle) translate(lang *language.Language, translationID string, args ...interface{}) string {
	if lang == nil {
		return translationID
	}

	translations := b.translations[lang.Tag]
	if translations == nil {
		translations = b.fallbackTranslations[lang.Tag]
		if translations == nil {
			return translationID
		}
	}

	translation := translations[translationID]
	if translation == nil {
		return translationID
	}

	var data interface{}
	var count interface{}
	if argc := len(args); argc > 0 {
		if isNumber(args[0]) {
			count = args[0]
			if argc > 1 {
				data = args[1]
			}
		} else {
			data = args[0]
		}
	}

	if count != nil {
		if data == nil {
			data = map[string]interface{}{"Count": count}
		} else {
			dataMap := toMap(data)
			dataMap["Count"] = count
			data = dataMap
		}
	} else {
		dataMap := toMap(data)
		if c, ok := dataMap["Count"]; ok {
			count = c
		}
	}

	p, _ := lang.Plural(count)
	template := translation.Template(p)
	if template == nil {
		return translationID
	}

	s := template.Execute(data)
	if s == "" {
		return translationID
	}
	return s
}

func isNumber(n interface{}) bool {
	switch n.(type) {
	case int, int8, int16, int32, int64, string:
		return true
	}
	return false
}

func toMap(input interface{}) map[string]interface{} {
	if data, ok := input.(map[string]interface{}); ok {
		return data
	}
	v := reflect.ValueOf(input)
	switch v.Kind() {
	case reflect.Ptr:
		return toMap(v.Elem().Interface())
	case reflect.Struct:
		return structToMap(v)
	default:
		return nil
	}
}

// Converts the top level of a struct to a map[string]interface{}.
// Code inspired by github.com/fatih/structs.
func structToMap(v reflect.Value) map[string]interface{} {
	out := make(map[string]interface{})
	t := v.Type()
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		if field.PkgPath != "" {
			// unexported field. skip.
			continue
		}
		out[field.Name] = v.FieldByName(field.Name).Interface()
	}
	return out
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 The btcsuite developers
// Copyright (c) 2015 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"runtime/pprof"
	"time"

	"github.com/decred/dcrd/limits"
)

var (
	cfg             *config
	shutdownChannel = make(chan struct{})
)

// winServiceMain is only invoked on Windows.  It detects when dcrd is running
// as a service and reacts accordingly.
var winServiceMain func() (bool, error)

// dcrdMain is the real main function for dcrd.  It is necessary to work around
// the fact that deferred functions do not run when os.Exit() is called.  The
// optional serverChan parameter is mainly used by the service code to be
// notified with the server once it is setup so it can gracefully stop it when
// requested from the service control manager.
func dcrdMain(serverChan chan<- *server) error {
	// Load configuration and parse command line.  This function also
	// initializes logging and configures it accordingly.
	tcfg, _, err := loadConfig()
	if err != nil {
		return err
	}
	cfg = tcfg
	defer backendLog.Flush()

	// Show version at startup.
	dcrdLog.Infof("Version %s", version())
	// Show dcrd home dir location
	dcrdLog.Debugf("Dcrd home dir: %s", cfg.DcrdHomeDir)

	// Enable http profiling server if requested.
	if cfg.Profile != "" {
		go func() {
			listenAddr := net.JoinHostPort("", cfg.Profile)
			dcrdLog.Infof("Creating profiling server "+
				"listening on %s", listenAddr)
			profileRedirect := http.RedirectHandler("/debug/pprof",
				http.StatusSeeOther)
			http.Handle("/", profileRedirect)
			err := http.ListenAndServe(listenAddr, nil)
			if err != nil {
				fatalf(err.Error())
			}
		}()
	}

	// Write cpu profile if requested.
	if cfg.CPUProfile != "" {
		f, err := os.Create(cfg.CPUProfile)
		if err != nil {
			dcrdLog.Errorf("Unable to create cpu profile: %v", err.Error())
			return err
		}
		pprof.StartCPUProfile(f)
		defer f.Close()
		defer pprof.StopCPUProfile()
	}

	// Write mem profile if requested.
	if cfg.MemProfile != "" {
		f, err := os.Create(cfg.MemProfile)
		if err != nil {
			dcrdLog.Errorf("Unable to create cpu profile: %v", err)
			return err
		}
		timer := time.NewTimer(time.Minute * 20) // 20 minutes
		go func() {
			<-timer.C
			pprof.WriteHeapProfile(f)
			f.Close()
		}()
	}

	// Perform upgrades to dcrd as new versions require it.
	if err := doUpgrades(); err != nil {
		dcrdLog.Errorf("%v", err)
		return err
	}

	// Load the block database.
	db, err := loadBlockDB()
	if err != nil {
		dcrdLog.Errorf("%v", err)
		return err
	}
	defer db.Close()

	if cfg.DropAddrIndex {
		dcrdLog.Info("Deleting entire addrindex.")
		err := db.PurgeAddrIndex()
		if err != nil {
			dcrdLog.Errorf("Unable to delete the addrindex: %v", err)
			return err
		}
		dcrdLog.Info("Successfully deleted addrindex, exiting")
		return nil
	}

	tmdb, err := loadTicketDB(db, activeNetParams.Params)
	if err != nil {
		dcrdLog.Errorf("%v", err)
		return err
	}
	defer func() {
		err := tmdb.Store(cfg.DataDir, "ticketdb.gob")
		if err != nil {
			dcrdLog.Errorf("Failed to store ticket database: %v", err.Error())
		}
	}()
	defer tmdb.Close()

	// Ensure the databases are sync'd and closed on Ctrl+C.
	addInterruptHandler(func() {
		dcrdLog.Infof("Gracefully shutting down the database...")
		db.RollbackClose()
	})

	// Create server and start it.
	server, err := newServer(cfg.Listeners, db, tmdb, activeNetParams.Params)
	if err != nil {
		// TODO(oga) this logging could do with some beautifying.
		dcrdLog.Errorf("Unable to start server on %v: %v",
			cfg.Listeners, err)
		return err
	}
	addInterruptHandler(func() {
		dcrdLog.Infof("Gracefully shutting down the server...")
		server.Stop()
		server.WaitForShutdown()
	})
	server.Start()
	if serverChan != nil {
		serverChan <- server
	}

	// Monitor for graceful server shutdown and signal the main goroutine
	// when done.  This is done in a separate goroutine rather than waiting
	// directly so the main goroutine can be signaled for shutdown by either
	// a graceful shutdown or from the main interrupt handler.  This is
	// necessary since the main goroutine must be kept running long enough
	// for the interrupt handler goroutine to finish.
	go func() {
		server.WaitForShutdown()
		srvrLog.Infof("Server shutdown complete")
		shutdownChannel <- struct{}{}
	}()

	// Wait for shutdown signal from either a graceful server stop or from
	// the interrupt handler.
	<-shutdownChannel
	dcrdLog.Info("Shutdown complete")
	return nil
}

func main() {
	// Use all processor cores.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Up some limits.
	if err := limits.SetLimits(); err != nil {
		fmt.Fprintf(os.Stderr, "failed to set limits: %v\n", err)
		os.Exit(1)
	}

	// Call serviceMain on Windows to handle running as a service.  When
	// the return isService flag is true, exit now since we ran as a
	// service.  Otherwise, just fall through to normal operation.
	if runtime.GOOS == "windows" {
		isService, err := winServiceMain()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		if isService {
			os.Exit(0)
		}
	}

	// Work around defer not working after os.Exit()
	if err := dcrdMain(nil); err != nil {
		os.Exit(1)
	}
}
<commit_msg>main: Limit garbage collection percentage. (#686) (#187)<commit_after>// Copyright (c) 2013-2014 The btcsuite developers
// Copyright (c) 2015 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"time"

	"github.com/decred/dcrd/limits"
)

var (
	cfg             *config
	shutdownChannel = make(chan struct{})
)

// winServiceMain is only invoked on Windows.  It detects when dcrd is running
// as a service and reacts accordingly.
var winServiceMain func() (bool, error)

// dcrdMain is the real main function for dcrd.  It is necessary to work around
// the fact that deferred functions do not run when os.Exit() is called.  The
// optional serverChan parameter is mainly used by the service code to be
// notified with the server once it is setup so it can gracefully stop it when
// requested from the service control manager.
func dcrdMain(serverChan chan<- *server) error {
	// Load configuration and parse command line.  This function also
	// initializes logging and configures it accordingly.
	tcfg, _, err := loadConfig()
	if err != nil {
		return err
	}
	cfg = tcfg
	defer backendLog.Flush()

	// Show version at startup.
	dcrdLog.Infof("Version %s", version())
	// Show dcrd home dir location
	dcrdLog.Debugf("Dcrd home dir: %s", cfg.DcrdHomeDir)

	// Enable http profiling server if requested.
	if cfg.Profile != "" {
		go func() {
			listenAddr := net.JoinHostPort("", cfg.Profile)
			dcrdLog.Infof("Creating profiling server "+
				"listening on %s", listenAddr)
			profileRedirect := http.RedirectHandler("/debug/pprof",
				http.StatusSeeOther)
			http.Handle("/", profileRedirect)
			err := http.ListenAndServe(listenAddr, nil)
			if err != nil {
				fatalf(err.Error())
			}
		}()
	}

	// Write cpu profile if requested.
	if cfg.CPUProfile != "" {
		f, err := os.Create(cfg.CPUProfile)
		if err != nil {
			dcrdLog.Errorf("Unable to create cpu profile: %v", err.Error())
			return err
		}
		pprof.StartCPUProfile(f)
		defer f.Close()
		defer pprof.StopCPUProfile()
	}

	// Write mem profile if requested.
	if cfg.MemProfile != "" {
		f, err := os.Create(cfg.MemProfile)
		if err != nil {
			dcrdLog.Errorf("Unable to create cpu profile: %v", err)
			return err
		}
		timer := time.NewTimer(time.Minute * 20) // 20 minutes
		go func() {
			<-timer.C
			pprof.WriteHeapProfile(f)
			f.Close()
		}()
	}

	// Perform upgrades to dcrd as new versions require it.
	if err := doUpgrades(); err != nil {
		dcrdLog.Errorf("%v", err)
		return err
	}

	// Load the block database.
	db, err := loadBlockDB()
	if err != nil {
		dcrdLog.Errorf("%v", err)
		return err
	}
	defer db.Close()

	if cfg.DropAddrIndex {
		dcrdLog.Info("Deleting entire addrindex.")
		err := db.PurgeAddrIndex()
		if err != nil {
			dcrdLog.Errorf("Unable to delete the addrindex: %v", err)
			return err
		}
		dcrdLog.Info("Successfully deleted addrindex, exiting")
		return nil
	}

	tmdb, err := loadTicketDB(db, activeNetParams.Params)
	if err != nil {
		dcrdLog.Errorf("%v", err)
		return err
	}
	defer func() {
		err := tmdb.Store(cfg.DataDir, "ticketdb.gob")
		if err != nil {
			dcrdLog.Errorf("Failed to store ticket database: %v", err.Error())
		}
	}()
	defer tmdb.Close()

	// Ensure the databases are sync'd and closed on Ctrl+C.
	addInterruptHandler(func() {
		dcrdLog.Infof("Gracefully shutting down the database...")
		db.RollbackClose()
	})

	// Create server and start it.
	server, err := newServer(cfg.Listeners, db, tmdb, activeNetParams.Params)
	if err != nil {
		// TODO(oga) this logging could do with some beautifying.
		dcrdLog.Errorf("Unable to start server on %v: %v",
			cfg.Listeners, err)
		return err
	}
	addInterruptHandler(func() {
		dcrdLog.Infof("Gracefully shutting down the server...")
		server.Stop()
		server.WaitForShutdown()
	})
	server.Start()
	if serverChan != nil {
		serverChan <- server
	}

	// Monitor for graceful server shutdown and signal the main goroutine
	// when done.  This is done in a separate goroutine rather than waiting
	// directly so the main goroutine can be signaled for shutdown by either
	// a graceful shutdown or from the main interrupt handler.  This is
	// necessary since the main goroutine must be kept running long enough
	// for the interrupt handler goroutine to finish.
	go func() {
		server.WaitForShutdown()
		srvrLog.Infof("Server shutdown complete")
		shutdownChannel <- struct{}{}
	}()

	// Wait for shutdown signal from either a graceful server stop or from
	// the interrupt handler.
	<-shutdownChannel
	dcrdLog.Info("Shutdown complete")
	return nil
}

func main() {
	// Use all processor cores.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Block and transaction processing can cause bursty allocations.  This
	// limits the garbage collector from excessively overallocating during
	// bursts.  This value was arrived at with the help of profiling live
	// usage.
	debug.SetGCPercent(10)

	// Up some limits.
	if err := limits.SetLimits(); err != nil {
		fmt.Fprintf(os.Stderr, "failed to set limits: %v\n", err)
		os.Exit(1)
	}

	// Call serviceMain on Windows to handle running as a service.  When
	// the return isService flag is true, exit now since we ran as a
	// service.  Otherwise, just fall through to normal operation.
	if runtime.GOOS == "windows" {
		isService, err := winServiceMain()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		if isService {
			os.Exit(0)
		}
	}

	// Work around defer not working after os.Exit()
	if err := dcrdMain(nil); err != nil {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package models

import (
	"testing"

	"gopkg.in/mgo.v2/bson"
)

// TestGetChallengeByIDSuccess tests for a successfully found ObjectID
func TestGetChallengeByIDSuccess(t *testing.T) {
	id := bson.ObjectIdHex("59a309ddf02210361b3b027f")

	challenge, err := GetChallengeByID(id)
	if err != nil {
		t.Errorf("Unable to retrieve challenge by ObjectID:\n %v", err)
	}

	if challenge.ID != id {
		t.Errorf("Challenge ID %v, is not equal to %v", challenge.ID, id)
	}
}

// TestGetChallengeByIDFailure tests that an error is returned when an ObjectID is not found
func TestGetChallengeByIDFailure(t *testing.T) {
	id := bson.ObjectIdHex("000000000000000000000000")

	_, err := GetChallengeByID(id)
	if err.Error() != "not found" {
		t.Errorf("Unable to throw error for ID:\n %v", err)
	}
}

func TestCreateChallengeSuccess(t *testing.T) {
	id := bson.NewObjectId()
	c := Challenge{
		ID: id,
	}
	if err := CreateChallenge(c); err != nil {
		t.Fatalf("Error creating a new test challenge:\n %v", err)
	}
	defer RemoveChallenge(id)
}

func TestCreateChallengeFailure(t *testing.T) {
	c := Challenge{
		ID: "fred",
	}
	if err := CreateChallenge(c); err == nil {
		t.Error("Did not handle error creating a new test challenge")
	}
}

func TestGetPendingChallengesSuccess(t *testing.T) {
	var id int64 = 1027935

	challenges, err := GetPendingChallenges(id)
	if err != nil {
		t.Errorf("Unable to retrieve pending challenges:\n %v", err)
	}

	if len(*challenges) <= 0 {
		t.Errorf("No pending challenges found for user %d", id)
	}
}

func TestGetActiveChallengesSuccess(t *testing.T) {
	var id int64 = 1027935

	challenges, err := GetActiveChallenges(id)
	if err != nil {
		t.Errorf("Unable to retrieve pending challenges:\n %v", err)
	}

	if len(*challenges) <= 0 {
		t.Errorf("No pending challenges found for user %d", id)
	}
}

func TestGetCompletedChallengesSuccess(t *testing.T) {
	var id int64 = 1027935

	challenges, err := GetCompletedChallenges(id)
	if err != nil {
		t.Errorf("Unable to retrieve pending challenges:\n %v", err)
	}

	if len(*challenges) <= 0 {
		t.Errorf("No pending challenges found for user %d", id)
	}
}
<commit_msg>add challenge model test<commit_after>package models

import (
	"testing"

	"gopkg.in/mgo.v2/bson"
)

// TestGetChallengeByIDSuccess tests for a successfully found ObjectID
func TestGetChallengeByIDSuccess(t *testing.T) {
	id := bson.ObjectIdHex("59a309ddf02210361b3b027f")

	challenge, err := GetChallengeByID(id)
	if err != nil {
		t.Errorf("Unable to retrieve challenge by ObjectID:\n %v", err)
	}

	if challenge.ID != id {
		t.Errorf("Challenge ID %v, is not equal to %v", challenge.ID, id)
	}
}

// TestGetChallengeByIDFailure tests that an error is returned when an ObjectID is not found
func TestGetChallengeByIDFailure(t *testing.T) {
	id := bson.ObjectIdHex("000000000000000000000000")

	_, err := GetChallengeByID(id)
	if err.Error() != "not found" {
		t.Errorf("Unable to throw error for ID:\n %v", err)
	}
}

func TestCreateChallengeSuccess(t *testing.T) {
	id := bson.NewObjectId()
	c := Challenge{
		ID: id,
	}
	if err := CreateChallenge(c); err != nil {
		t.Fatalf("Error creating a new test challenge:\n %v", err)
	}
	defer RemoveChallenge(id)
}

func TestCreateChallengeFailure(t *testing.T) {
	c := Challenge{
		ID: "fred",
	}
	if err := CreateChallenge(c); err == nil {
		t.Error("Did not handle error creating a new test challenge")
	}
}

func TestGetPendingChallengesSuccess(t *testing.T) {
	var id int64 = 1027935

	challenges, err := GetPendingChallenges(id)
	if err != nil {
		t.Errorf("Unable to retrieve pending challenges:\n %v", err)
	}

	if len(*challenges) <= 0 {
		t.Errorf("No pending challenges found for user %d", id)
	}
}

func TestGetActiveChallengesSuccess(t *testing.T) {
	var id int64 = 1027935

	challenges, err := GetActiveChallenges(id)
	if err != nil {
		t.Errorf("Unable to retrieve pending challenges:\n %v", err)
	}

	if len(*challenges) <= 0 {
		t.Errorf("No pending challenges found for user %d", id)
	}
}

func TestGetCompletedChallengesSuccess(t *testing.T) {
	var id int64 = 1027935

	challenges, err := GetCompletedChallenges(id)
	if err != nil {
		t.Errorf("Unable to retrieve pending challenges:\n %v", err)
	}

	if len(*challenges) <= 0 {
		t.Errorf("No pending challenges found for user %d", id)
	}
}

func TestGetAllChallengesSuccess(t *testing.T) {
	var id int64 = 1027935

	challenges, err := GetAllChallenges(id)
	if err != nil {
		t.Errorf("Unable to retrieve challenges:\n %v", err)
	}

	if len(*challenges) <= 0 {
		t.Errorf("No challenges found for user %d", id)
	}
}
<|endoftext|>


<commit_before>package models

import (
	"time"

	"bytes"
	"fmt"
	"github.com/astaxie/beego"
	"github.com/astaxie/beego/orm"
	"github.com/lifei6671/mindoc/conf"
	"strings"
	"os"
	"path/filepath"
	"strconv"
	"github.com/PuerkitoBio/goquery"
)

// Document struct.
type Document struct {
	DocumentId   int    `orm:"pk;auto;unique;column(document_id)" json:"doc_id"`
	DocumentName string `orm:"column(document_name);size(500)" json:"doc_name"`
	// Identify 文档唯一标识
	Identify  string `orm:"column(identify);size(100);index;null;default(null)" json:"identify"`
	BookId    int    `orm:"column(book_id);type(int);index" json:"book_id"`
	ParentId  int    `orm:"column(parent_id);type(int);index;default(0)" json:"parent_id"`
	OrderSort int    `orm:"column(order_sort);default(0);type(int);index" json:"order_sort"`
	// Markdown markdown格式文档.
	Markdown string `orm:"column(markdown);type(text);null" json:"markdown"`
	// Release 发布后的Html格式内容.
	Release string `orm:"column(release);type(text);null" json:"release"`
	// Content 未发布的 Html 格式内容.
	Content    string        `orm:"column(content);type(text);null" json:"content"`
	CreateTime time.Time     `orm:"column(create_time);type(datetime);auto_now_add" json:"create_time"`
	MemberId   int           `orm:"column(member_id);type(int)" json:"member_id"`
	ModifyTime time.Time     `orm:"column(modify_time);type(datetime);auto_now" json:"modify_time"`
	ModifyAt   int           `orm:"column(modify_at);type(int)" json:"-"`
	Version    int64         `orm:"type(bigint);column(version)" json:"version"`
	AttachList []*Attachment `orm:"-" json:"attach"`
}

// TableName 获取对应数据库表名.
func (m *Document) TableName() string {
	return "documents"
}

// TableEngine 获取数据使用的引擎.
func (m *Document) TableEngine() string {
	return "INNODB"
}

func (m *Document) TableNameWithPrefix() string {
	return conf.GetDatabasePrefix() + m.TableName()
}

func NewDocument() *Document {
	return &Document{
		Version: time.Now().Unix(),
	}
}

//根据文档ID查询指定文档.
func (m *Document) Find(id int) (*Document, error) {
	if id <= 0 {
		return m, ErrInvalidParameter
	}
	o := orm.NewOrm()

	err := o.QueryTable(m.TableNameWithPrefix()).Filter("document_id", id).One(m)

	if err == orm.ErrNoRows {
		return m, ErrDataNotExist
	}
	return m, nil
}

//插入和更新文档.
func (m *Document) InsertOrUpdate(cols ...string) error {
	o := orm.NewOrm()

	if m.DocumentId > 0 {
		_, err := o.Update(m)
		return err
	} else {
		_, err := o.Insert(m)
		NewBook().ResetDocumentNumber(m.BookId)
		return err
	}
}

//根据指定字段查询一条文档.
func (m *Document) FindByFieldFirst(field string, v interface{}) (*Document, error) {
	o := orm.NewOrm()

	err := o.QueryTable(m.TableNameWithPrefix()).Filter(field, v).One(m)

	return m, err
}

//递归删除一个文档.
func (m *Document) RecursiveDocument(doc_id int) error {

	o := orm.NewOrm()

	if doc, err := m.Find(doc_id); err == nil {
		o.Delete(doc)
		NewDocumentHistory().Clear(doc.DocumentId)
	}

	var docs []*Document

	_, err := o.QueryTable(m.TableNameWithPrefix()).Filter("parent_id", doc_id).All(&docs)

	if err != nil {
		beego.Error("RecursiveDocument => ", err)
		return err
	}

	for _, item := range docs {
		doc_id := item.DocumentId
		o.QueryTable(m.TableNameWithPrefix()).Filter("document_id", doc_id).Delete()
		m.RecursiveDocument(doc_id)
	}

	return nil
}

//发布文档
func (m *Document) ReleaseContent(bookId int) {

	o := orm.NewOrm()

	var docs []*Document
	_, err := o.QueryTable(m.TableNameWithPrefix()).Filter("book_id", bookId).All(&docs, "document_id", "content")

	if err != nil {
		beego.Error("发布失败 => ", err)
		return
	}
	for _, item := range docs {
		if item.Content != "" {
			item.Release = item.Content
			bufio := bytes.NewReader([]byte(item.Content))
			//解析文档中非本站的链接，并设置为新窗口打开
			if content, err := goquery.NewDocumentFromReader(bufio);err == nil {

				content.Find("a").Each(func(i int, contentSelection *goquery.Selection) {
					if src, ok := contentSelection.Attr("href"); ok{
						if strings.HasPrefix(src, "http://") || strings.HasPrefix(src,"https://") {
							if conf.BaseUrl != "" && strings.Index(src,conf.BaseUrl) != 0 {
								contentSelection.SetAttr("target", "_blank")
								if html, err := content.Html();err == nil {
									item.Release = html
								}
							}
						}

					}
				})
			}
		}

		attachList, err := NewAttachment().FindListByDocumentId(item.DocumentId)
		if err == nil && len(attachList) > 0 {
			content := bytes.NewBufferString("
附件
")
			for _, attach := range attachList {
				if strings.HasPrefix(attach.HttpPath, "/") {
					attach.HttpPath = strings.TrimSuffix(beego.AppConfig.DefaultString("baseurl", ""), "/") + attach.HttpPath
				}
				li := fmt.Sprintf("
%s
", attach.HttpPath, attach.FileName, attach.FileName)

				content.WriteString(li)
			}
			content.WriteString("

")
			item.Release += content.String()
		}
		_, err = o.Update(item, "release")
		if err != nil {
			beego.Error(fmt.Sprintf("发布失败 => %+v", item), err)
		}else {
			os.RemoveAll(filepath.Join(conf.WorkingDirectory,"uploads","books",strconv.Itoa(bookId)))
		}
	}
}

//根据项目ID查询文档列表.
func (m *Document) FindListByBookId(book_id int) (docs []*Document, err error) {
	o := orm.NewOrm()

	_, err = o.QueryTable(m.TableNameWithPrefix()).Filter("book_id", book_id).OrderBy("order_sort").All(&docs)

	return
}
<commit_msg>实现外链新窗口打开<commit_after>package models

import (
	"time"

	"bytes"
	"fmt"
	"github.com/astaxie/beego"
	"github.com/astaxie/beego/orm"
	"github.com/lifei6671/mindoc/conf"
	"strings"
	"os"
	"path/filepath"
	"strconv"
	"github.com/PuerkitoBio/goquery"
)

// Document struct.
type Document struct {
	DocumentId   int    `orm:"pk;auto;unique;column(document_id)" json:"doc_id"`
	DocumentName string `orm:"column(document_name);size(500)" json:"doc_name"`
	// Identify 文档唯一标识
	Identify  string `orm:"column(identify);size(100);index;null;default(null)" json:"identify"`
	BookId    int    `orm:"column(book_id);type(int);index" json:"book_id"`
	ParentId  int    `orm:"column(parent_id);type(int);index;default(0)" json:"parent_id"`
	OrderSort int    `orm:"column(order_sort);default(0);type(int);index" json:"order_sort"`
	// Markdown markdown格式文档.
	Markdown string `orm:"column(markdown);type(text);null" json:"markdown"`
	// Release 发布后的Html格式内容.
	Release string `orm:"column(release);type(text);null" json:"release"`
	// Content 未发布的 Html 格式内容.
	Content    string        `orm:"column(content);type(text);null" json:"content"`
	CreateTime time.Time     `orm:"column(create_time);type(datetime);auto_now_add" json:"create_time"`
	MemberId   int           `orm:"column(member_id);type(int)" json:"member_id"`
	ModifyTime time.Time     `orm:"column(modify_time);type(datetime);auto_now" json:"modify_time"`
	ModifyAt   int           `orm:"column(modify_at);type(int)" json:"-"`
	Version    int64         `orm:"type(bigint);column(version)" json:"version"`
	AttachList []*Attachment `orm:"-" json:"attach"`
}

// TableName 获取对应数据库表名.
func (m *Document) TableName() string {
	return "documents"
}

// TableEngine 获取数据使用的引擎.
func (m *Document) TableEngine() string {
	return "INNODB"
}

func (m *Document) TableNameWithPrefix() string {
	return conf.GetDatabasePrefix() + m.TableName()
}

func NewDocument() *Document {
	return &Document{
		Version: time.Now().Unix(),
	}
}

//根据文档ID查询指定文档.
func (m *Document) Find(id int) (*Document, error) {
	if id <= 0 {
		return m, ErrInvalidParameter
	}
	o := orm.NewOrm()

	err := o.QueryTable(m.TableNameWithPrefix()).Filter("document_id", id).One(m)

	if err == orm.ErrNoRows {
		return m, ErrDataNotExist
	}
	return m, nil
}

//插入和更新文档.
func (m *Document) InsertOrUpdate(cols ...string) error {
	o := orm.NewOrm()

	if m.DocumentId > 0 {
		_, err := o.Update(m)
		return err
	} else {
		_, err := o.Insert(m)
		NewBook().ResetDocumentNumber(m.BookId)
		return err
	}
}

//根据指定字段查询一条文档.
func (m *Document) FindByFieldFirst(field string, v interface{}) (*Document, error) {
	o := orm.NewOrm()

	err := o.QueryTable(m.TableNameWithPrefix()).Filter(field, v).One(m)

	return m, err
}

//递归删除一个文档.
func (m *Document) RecursiveDocument(doc_id int) error {

	o := orm.NewOrm()

	if doc, err := m.Find(doc_id); err == nil {
		o.Delete(doc)
		NewDocumentHistory().Clear(doc.DocumentId)
	}

	var docs []*Document

	_, err := o.QueryTable(m.TableNameWithPrefix()).Filter("parent_id", doc_id).All(&docs)

	if err != nil {
		beego.Error("RecursiveDocument => ", err)
		return err
	}

	for _, item := range docs {
		doc_id := item.DocumentId
		o.QueryTable(m.TableNameWithPrefix()).Filter("document_id", doc_id).Delete()
		m.RecursiveDocument(doc_id)
	}

	return nil
}

//发布文档
func (m *Document) ReleaseContent(bookId int) {

	o := orm.NewOrm()

	var docs []*Document
	_, err := o.QueryTable(m.TableNameWithPrefix()).Filter("book_id", bookId).All(&docs, "document_id", "content")

	if err != nil {
		beego.Error("发布失败 => ", err)
		return
	}
	for _, item := range docs {
		if item.Content != "" {
			item.Release = item.Content
			bufio := bytes.NewReader([]byte(item.Content))
			//解析文档中非本站的链接，并设置为新窗口打开
			if content, err := goquery.NewDocumentFromReader(bufio);err == nil {

				content.Find("a").Each(func(i int, contentSelection *goquery.Selection) {
					if src, ok := contentSelection.Attr("href"); ok{
						if strings.HasPrefix(src, "http://") || strings.HasPrefix(src,"https://") {
							beego.Info(src,conf.BaseUrl,strings.HasPrefix(src,conf.BaseUrl))
							if conf.BaseUrl != "" && !strings.HasPrefix(src,conf.BaseUrl) {
								contentSelection.SetAttr("target", "_blank")
								if html, err := content.Html();err == nil {
									item.Release = html
								}
							}
						}

					}
				})
			}
		}

		attachList, err := NewAttachment().FindListByDocumentId(item.DocumentId)
		if err == nil && len(attachList) > 0 {
			content := bytes.NewBufferString("
附件
")
			for _, attach := range attachList {
				if strings.HasPrefix(attach.HttpPath, "/") {
					attach.HttpPath = strings.TrimSuffix(conf.BaseUrl, "/") + attach.HttpPath
				}
				li := fmt.Sprintf("
%s
", attach.HttpPath, attach.FileName, attach.FileName)

				content.WriteString(li)
			}
			content.WriteString("

")
			item.Release += content.String()
		}
		_, err = o.Update(item, "release")
		if err != nil {
			beego.Error(fmt.Sprintf("发布失败 => %+v", item), err)
		}else {
			os.RemoveAll(filepath.Join(conf.WorkingDirectory,"uploads","books",strconv.Itoa(bookId)))
		}
	}
}

//根据项目ID查询文档列表.
func (m *Document) FindListByBookId(book_id int) (docs []*Document, err error) {
	o := orm.NewOrm()

	_, err = o.QueryTable(m.TableNameWithPrefix()).Filter("book_id", book_id).OrderBy("order_sort").All(&docs)

	return
}
<|endoftext|>


<commit_before>//go:generate mapstructure-to-hcl2 -type Config

package null

import (
	"fmt"

	"github.com/hashicorp/packer/common"
	"github.com/hashicorp/packer/helper/communicator"
	"github.com/hashicorp/packer/helper/config"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/template/interpolate"
)

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	CommConfig communicator.Config `mapstructure:",squash"`
}

func (c *Config) Prepare(raws ...interface{}) ([]string, error) {

	err := config.Decode(&c, &config.DecodeOpts{
		Interpolate:       true,
		InterpolateFilter: &interpolate.RenderFilter{},
	}, raws...)
	if err != nil {
		return nil, err
	}

	var errs *packer.MultiError
	if es := c.CommConfig.Prepare(nil); len(es) > 0 {
		errs = packer.MultiErrorAppend(errs, es...)
	}

	if c.CommConfig.Type != "none" {
		if c.CommConfig.Host() == "" {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("a Host must be specified, please reference your communicator documentation"))
		}

		if c.CommConfig.User() == "" {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("a Username must be specified, please reference your communicator documentation"))
		}

		if !c.CommConfig.SSHAgentAuth && c.CommConfig.Password() == "" && c.CommConfig.SSHPrivateKeyFile == "" {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("one authentication method must be specified, please reference your communicator documentation"))
		}

		if (c.CommConfig.SSHAgentAuth &&
			(c.CommConfig.SSHPassword != "" || c.CommConfig.SSHPrivateKeyFile != "")) ||
			(c.CommConfig.SSHPassword != "" && c.CommConfig.SSHPrivateKeyFile != "") {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("only one of ssh_agent_auth, ssh_password, and ssh_private_key_file must be specified"))

		}
	}

	if errs != nil && len(errs.Errors) > 0 {
		return nil, errs
	}

	return nil, nil
}
<commit_msg>fix config decode<commit_after>//go:generate mapstructure-to-hcl2 -type Config

package null

import (
	"fmt"

	"github.com/hashicorp/packer/common"
	"github.com/hashicorp/packer/helper/communicator"
	"github.com/hashicorp/packer/helper/config"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/template/interpolate"
)

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	CommConfig communicator.Config `mapstructure:",squash"`
}

func (c *Config) Prepare(raws ...interface{}) ([]string, error) {

	err := config.Decode(c, &config.DecodeOpts{
		Interpolate:       true,
		InterpolateFilter: &interpolate.RenderFilter{},
	}, raws...)
	if err != nil {
		return nil, err
	}

	var errs *packer.MultiError
	if es := c.CommConfig.Prepare(nil); len(es) > 0 {
		errs = packer.MultiErrorAppend(errs, es...)
	}

	if c.CommConfig.Type != "none" {
		if c.CommConfig.Host() == "" {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("a Host must be specified, please reference your communicator documentation"))
		}

		if c.CommConfig.User() == "" {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("a Username must be specified, please reference your communicator documentation"))
		}

		if !c.CommConfig.SSHAgentAuth && c.CommConfig.Password() == "" && c.CommConfig.SSHPrivateKeyFile == "" {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("one authentication method must be specified, please reference your communicator documentation"))
		}

		if (c.CommConfig.SSHAgentAuth &&
			(c.CommConfig.SSHPassword != "" || c.CommConfig.SSHPrivateKeyFile != "")) ||
			(c.CommConfig.SSHPassword != "" && c.CommConfig.SSHPrivateKeyFile != "") {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("only one of ssh_agent_auth, ssh_password, and ssh_private_key_file must be specified"))

		}
	}

	if errs != nil && len(errs.Errors) > 0 {
		return nil, errs
	}

	return nil, nil
}
<|endoftext|>


<commit_before>package http

import (
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"gopkg.in/gin-gonic/gin.v1"
	"github.com/bitly/go-simplejson"
	"github.com/satori/go.uuid"

	ogin "github.com/Cepave/open-falcon-backend/common/gin"
	nqmDb "github.com/Cepave/open-falcon-backend/common/db/nqm"
	commonModel "github.com/Cepave/open-falcon-backend/common/model"

	"github.com/Cepave/open-falcon-backend/modules/query/nqm"
	model "github.com/Cepave/open-falcon-backend/modules/query/model/nqm"
	dsl "github.com/Cepave/open-falcon-backend/modules/query/dsl/nqm_parser"
	metricDsl "github.com/Cepave/open-falcon-backend/modules/query/dsl/metric_parser"
)

var nqmService *nqm.ServiceController

// Although these services use Gin framework, the configuration depends on "http.listen" property,
// not "gin_http.listen"
func configNqmRoutes() {
	nqmService = nqm.GetDefaultServiceController()
	nqmService.Init()

	http.Handle("/nqm/", getGinRouter())
}

func getGinRouter() *gin.Engine {
	engine := ogin.NewDefaultJsonEngine(&ogin.GinConfig{ Mode: gin.ReleaseMode })

	engine.GET("/nqm/icmp/list/by-provinces", listIcmpByProvinces)
	engine.GET("/nqm/icmp/province/:province_id/list/by-targets", listIcmpByTargetsForAProvince)
	engine.GET("/nqm/province/:province_id/agents", listEffectiveAgentsInProvince)

	compoundReport := engine.Group("/nqm/icmp/compound-report")
	{
		compoundReport.GET("", outputCompondReportOfIcmp)
		compoundReport.POST("", buildQueryOfIcmp)

		compoundReport.GET("/query/:query_id", getQueryContentOfIcmp)
	}

	return engine
}

func buildQueryOfIcmp(context *gin.Context) {
	compoundQuery, err := buildCompoundQueryOfIcmp(context)

	/**
	 * Output status(400) for error of metric DSL
	 */
	if err != nil {
		switch err.(type) {
		case dslError:
			context.JSON(http.StatusBadRequest, err)
			return
		}
	}
	// :~)

	compoundQuery.SetupDefault()
	query := nqm.BuildQuery(compoundQuery)
	context.JSON(http.StatusOK, query.ToJson())
}

func getQueryContentOfIcmp(context *gin.Context) {
	compoundQuery, hasQuery := loadCompoundQueryByUuid(
		context,
		context.Param("query_id"), "/nqm/icmp/compound-report/query/%s",
	)
	if !hasQuery {
		return
	}

	context.JSON(http.StatusOK, nqm.ToQueryDetail(compoundQuery))
}
func outputCompondReportOfIcmp(context *gin.Context) {
	compoundQuery, hasQuery := loadCompoundQueryByUuid(
		context,
		context.Query("query_id"), "/nqm/icmp/compound-report?query_id=%s",
	)
	if !hasQuery {
		return
	}

	/**
	 * Set-up paging
	 */
	paging := ogin.PagingByHeader(
		context,
		&commonModel.Paging {
			Size: 500,
			Position: 1,
		},
	)

	result := nqm.LoadIcmpRecordsOfCompoundQuery(compoundQuery, paging)
	ogin.HeaderWithPaging(context, paging)
	// :~)

	context.JSON(http.StatusOK, result)
}

func loadCompoundQueryByUuid(context *gin.Context, queryId string, errorFormatter string) (*model.CompoundQuery, bool) {
	uuidValue := uuid.FromStringOrNil(queryId)

	var showNotFound = func() {
		context.JSON(
			http.StatusNotFound,
			map[string] interface{} {
			  "http_status": http.StatusNotFound,
			  "uri": fmt.Sprintf(errorFormatter, queryId),
			  "error_code": 1,
			  "error_message": "Query id cannot be fetched",
			},
		)
	}

	if uuidValue == uuid.Nil {
		showNotFound()
		return nil, false
	}

	compoundQuery := nqm.GetCompoundQueryByUuid(uuidValue)
	if compoundQuery == nil {
		showNotFound()
		return nil, false
	}

	return compoundQuery, true
}

type dslError struct {
	ErrorCode int `json:"error_code"`
	Message string `json:"error_message"`
}

func (e dslError) Error() string {
	return e.Message
}

// Parses the JSON to query object and checks values
func buildCompoundQueryOfIcmp(context *gin.Context) (*model.CompoundQuery, error) {
	query := model.NewCompoundQuery()
	jsonErr := context.BindJSON(query)
	if jsonErr != nil {
		return nil, jsonErr
	}

	_, parseError := metricDsl.ParseToMetricFilter(query.Filters.Metrics)
	if parseError != nil {
		return nil, dslError {
			1, parseError.Error(),
		}
	}

	return query, nil
}

type resultWithDsl struct {
	queryParams *dsl.QueryParams
	resultData interface{}
}

func (result *resultWithDsl) MarshalJSON() ([]byte, error) {
	jsonObject := simplejson.New()

	jsonObject.SetPath([]string{ "dsl", "start_time" }, result.queryParams.StartTime.Unix())
	jsonObject.SetPath([]string{ "dsl", "end_time" }, result.queryParams.EndTime.Unix())
	jsonObject.Set("result", result.resultData)

	return jsonObject.MarshalJSON()
}

// Lists agents(grouped by city) for a province
func listEffectiveAgentsInProvince(context *gin.Context) {
	provinceId, err := strconv.ParseInt(context.Param("province_id"), 10, 16)
	if err != nil {
		panic(err)
	}

	context.JSON(
		http.StatusOK,
		nqmDb.LoadEffectiveAgentsInProvince(int16(provinceId)),
	)
}

// Lists statistics data of ICMP, which would be grouped by provinces
func listIcmpByProvinces(context *gin.Context) {
	dslParams, isValid := processDslAndOutputError(context, context.Query("dsl"))
	if !isValid {
		return
	}

	context.JSON(
		http.StatusOK,
		&resultWithDsl{
			queryParams: dslParams,
			resultData: nqmService.ListByProvinces(dslParams),
		},
	)
}

// Lists data of targets, which would be grouped by cities
func listIcmpByTargetsForAProvince(context *gin.Context) {
	dslParams, isValid := processDslAndOutputError(context, context.Query("dsl"))
	if !isValid {
		return
	}

	dslParams.AgentFilter.MatchProvinces = make([]string, 0) // Ignores the province of agent

	provinceId, _ := strconv.ParseInt(context.Param("province_id"), 10, 16)
	dslParams.AgentFilterById.MatchProvinces = []int16 { int16(provinceId) } // Use the id as the filter of agent

	if agentId, parseErrForAgentId := strconv.ParseInt(context.Query("agent_id"), 10, 16)
		parseErrForAgentId == nil {
		dslParams.AgentFilterById.MatchIds = []int32 { int32(agentId) } // Set the filter by agent's id
	} else if cityId, parseErrForCityId := strconv.ParseInt(context.Query("city_id_of_agent"), 10, 16)
		parseErrForCityId == nil {
		dslParams.AgentFilterById.MatchCities = []int16 { int16(cityId) } // Set the filter by city's id
	}

	context.JSON(
		http.StatusOK,
		&resultWithDsl{
			queryParams: dslParams,
			resultData: nqmService.ListTargetsWithCityDetail(dslParams),
		},
	)
}

type jsonDslError struct {
	Code int `json:"error_code"`
	Message string `json:"error_message"`
}
func outputDslError(context *gin.Context, err error) {
	context.JSON(
		http.StatusBadRequest,
		&jsonDslError {
			Code: 1,
			Message: err.Error(),
		},
	)
}

const (
	defaultDaysForTimeRange = 7
	after7Days = defaultDaysForTimeRange * 24 * time.Hour
	before7Days = after7Days * -1
)

// Process DSL and output error
// Returns: true if the DSL is valid
func processDslAndOutputError(context *gin.Context, dslText string) (*dsl.QueryParams, bool) {
	dslParams, err := processDsl(dslText)
	if err == nil {
		return dslParams, true
	}

	context.JSON(
		http.StatusBadRequest,
		&struct {
			Code int `json:"error_code"`
			Message string `json:"error_message"`
		} {
			Code: 1,
			Message: err.Error(),
		},
	)

	return nil, false
}

// The query of DSL would be inner-province(used for phase 1)
func processDsl(dslParams string) (*dsl.QueryParams, error) {
	strNqmDsl := strings.TrimSpace(dslParams)

	/**
	 * If any of errors for parsing DSL
	 */
	paramSetters, parseError := dsl.Parse(
		"Query.nqmdsl", []byte(strNqmDsl),
	)
	if parseError != nil {
		return nil, parseError
	}
	// :~)

	queryParams := dsl.NewQueryParams()
	queryParams.SetUpParams(paramSetters)

	setupTimeRange(queryParams)
	setupInnerProvince(queryParams)

	paramsError := queryParams.CheckRationalOfParameters()
	if paramsError != nil {
		return nil, paramsError
	}

	return queryParams, nil
}

// Sets-up the time range with provided-or-not value of parameters
// 1. Without any parameter of time range
// 2. Has only start time
// 3. Has only end time
func setupTimeRange(queryParams *dsl.QueryParams) {
	if queryParams.StartTime.IsZero() && queryParams.EndTime.IsZero() {
		now := time.Now()

		queryParams.StartTime = now.Add(before7Days) // Include 7 days before
		queryParams.EndTime = now.Add(24 * time.Hour) // Include today
		return
	}

	if queryParams.StartTime.IsZero() && !queryParams.EndTime.IsZero() {
		queryParams.StartTime = queryParams.EndTime.Add(before7Days)
		return
	}

	if !queryParams.StartTime.IsZero() && queryParams.EndTime.IsZero() {
		queryParams.EndTime = queryParams.StartTime.Add(after7Days)
		return
	}

	if queryParams.StartTime.Unix() == queryParams.EndTime.Unix() {
		queryParams.EndTime = queryParams.StartTime.Add(24 * time.Hour)
	}
}

/**
 * !IMPORTANT!
 * This default value is just used in phase 1 funcion of NQM reporting(inner-province)
 */
func setupInnerProvince(queryParams *dsl.QueryParams) {
	queryParams.ProvinceRelation = model.SameValue
}
<commit_msg>[OWL-1167] Fix the response for empty body of compound query<commit_after>package http

import (
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"
	"time"

	"gopkg.in/gin-gonic/gin.v1"
	"github.com/bitly/go-simplejson"
	"github.com/satori/go.uuid"

	ogin "github.com/Cepave/open-falcon-backend/common/gin"
	nqmDb "github.com/Cepave/open-falcon-backend/common/db/nqm"
	commonModel "github.com/Cepave/open-falcon-backend/common/model"

	"github.com/Cepave/open-falcon-backend/modules/query/nqm"
	model "github.com/Cepave/open-falcon-backend/modules/query/model/nqm"
	dsl "github.com/Cepave/open-falcon-backend/modules/query/dsl/nqm_parser"
	metricDsl "github.com/Cepave/open-falcon-backend/modules/query/dsl/metric_parser"
)

var nqmService *nqm.ServiceController

// Although these services use Gin framework, the configuration depends on "http.listen" property,
// not "gin_http.listen"
func configNqmRoutes() {
	nqmService = nqm.GetDefaultServiceController()
	nqmService.Init()

	http.Handle("/nqm/", getGinRouter())
}

func getGinRouter() *gin.Engine {
	engine := ogin.NewDefaultJsonEngine(&ogin.GinConfig{ Mode: gin.ReleaseMode })

	engine.GET("/nqm/icmp/list/by-provinces", listIcmpByProvinces)
	engine.GET("/nqm/icmp/province/:province_id/list/by-targets", listIcmpByTargetsForAProvince)
	engine.GET("/nqm/province/:province_id/agents", listEffectiveAgentsInProvince)

	compoundReport := engine.Group("/nqm/icmp/compound-report")
	{
		compoundReport.GET("", outputCompondReportOfIcmp)
		compoundReport.POST("", buildQueryOfIcmp)

		compoundReport.GET("/query/:query_id", getQueryContentOfIcmp)
	}

	return engine
}

func buildQueryOfIcmp(context *gin.Context) {
	compoundQuery, err := buildCompoundQueryOfIcmp(context)

	/**
	 * Output status(400) for error of metric DSL
	 */
	if err != nil {
		switch err.(type) {
		case dslError:
			context.JSON(http.StatusBadRequest, err)
		default:
			panic(err)
		}

		return
	}
	// :~)

	compoundQuery.SetupDefault()
	query := nqm.BuildQuery(compoundQuery)
	context.JSON(http.StatusOK, query.ToJson())
}

func getQueryContentOfIcmp(context *gin.Context) {
	compoundQuery, hasQuery := loadCompoundQueryByUuid(
		context,
		context.Param("query_id"), "/nqm/icmp/compound-report/query/%s",
	)
	if !hasQuery {
		return
	}

	context.JSON(http.StatusOK, nqm.ToQueryDetail(compoundQuery))
}
func outputCompondReportOfIcmp(context *gin.Context) {
	compoundQuery, hasQuery := loadCompoundQueryByUuid(
		context,
		context.Query("query_id"), "/nqm/icmp/compound-report?query_id=%s",
	)
	if !hasQuery {
		return
	}

	/**
	 * Set-up paging
	 */
	paging := ogin.PagingByHeader(
		context,
		&commonModel.Paging {
			Size: 500,
			Position: 1,
		},
	)

	result := nqm.LoadIcmpRecordsOfCompoundQuery(compoundQuery, paging)
	ogin.HeaderWithPaging(context, paging)
	// :~)

	context.JSON(http.StatusOK, result)
}

func loadCompoundQueryByUuid(context *gin.Context, queryId string, errorFormatter string) (*model.CompoundQuery, bool) {
	uuidValue := uuid.FromStringOrNil(queryId)

	var showNotFound = func() {
		context.JSON(
			http.StatusNotFound,
			map[string] interface{} {
			  "http_status": http.StatusNotFound,
			  "uri": fmt.Sprintf(errorFormatter, queryId),
			  "error_code": 1,
			  "error_message": "Query id cannot be fetched",
			},
		)
	}

	if uuidValue == uuid.Nil {
		showNotFound()
		return nil, false
	}

	compoundQuery := nqm.GetCompoundQueryByUuid(uuidValue)
	if compoundQuery == nil {
		showNotFound()
		return nil, false
	}

	return compoundQuery, true
}

type dslError struct {
	ErrorCode int `json:"error_code"`
	Message string `json:"error_message"`
}

func (e dslError) Error() string {
	return e.Message
}

// Parses the JSON to query object and checks values
func buildCompoundQueryOfIcmp(context *gin.Context) (*model.CompoundQuery, error) {
	query := model.NewCompoundQuery()

	jsonErr := context.BindJSON(query)
	if jsonErr == io.EOF {
		query.UnmarshalJSON([]byte("{}"))
	} else if jsonErr != nil {
		return nil, jsonErr
	}

	_, parseError := metricDsl.ParseToMetricFilter(query.Filters.Metrics)
	if parseError != nil {
		return nil, dslError {
			1, parseError.Error(),
		}
	}

	return query, nil
}

type resultWithDsl struct {
	queryParams *dsl.QueryParams
	resultData interface{}
}

func (result *resultWithDsl) MarshalJSON() ([]byte, error) {
	jsonObject := simplejson.New()

	jsonObject.SetPath([]string{ "dsl", "start_time" }, result.queryParams.StartTime.Unix())
	jsonObject.SetPath([]string{ "dsl", "end_time" }, result.queryParams.EndTime.Unix())
	jsonObject.Set("result", result.resultData)

	return jsonObject.MarshalJSON()
}

// Lists agents(grouped by city) for a province
func listEffectiveAgentsInProvince(context *gin.Context) {
	provinceId, err := strconv.ParseInt(context.Param("province_id"), 10, 16)
	if err != nil {
		panic(err)
	}

	context.JSON(
		http.StatusOK,
		nqmDb.LoadEffectiveAgentsInProvince(int16(provinceId)),
	)
}

// Lists statistics data of ICMP, which would be grouped by provinces
func listIcmpByProvinces(context *gin.Context) {
	dslParams, isValid := processDslAndOutputError(context, context.Query("dsl"))
	if !isValid {
		return
	}

	context.JSON(
		http.StatusOK,
		&resultWithDsl{
			queryParams: dslParams,
			resultData: nqmService.ListByProvinces(dslParams),
		},
	)
}

// Lists data of targets, which would be grouped by cities
func listIcmpByTargetsForAProvince(context *gin.Context) {
	dslParams, isValid := processDslAndOutputError(context, context.Query("dsl"))
	if !isValid {
		return
	}

	dslParams.AgentFilter.MatchProvinces = make([]string, 0) // Ignores the province of agent

	provinceId, _ := strconv.ParseInt(context.Param("province_id"), 10, 16)
	dslParams.AgentFilterById.MatchProvinces = []int16 { int16(provinceId) } // Use the id as the filter of agent

	if agentId, parseErrForAgentId := strconv.ParseInt(context.Query("agent_id"), 10, 16)
		parseErrForAgentId == nil {
		dslParams.AgentFilterById.MatchIds = []int32 { int32(agentId) } // Set the filter by agent's id
	} else if cityId, parseErrForCityId := strconv.ParseInt(context.Query("city_id_of_agent"), 10, 16)
		parseErrForCityId == nil {
		dslParams.AgentFilterById.MatchCities = []int16 { int16(cityId) } // Set the filter by city's id
	}

	context.JSON(
		http.StatusOK,
		&resultWithDsl{
			queryParams: dslParams,
			resultData: nqmService.ListTargetsWithCityDetail(dslParams),
		},
	)
}

type jsonDslError struct {
	Code int `json:"error_code"`
	Message string `json:"error_message"`
}
func outputDslError(context *gin.Context, err error) {
	context.JSON(
		http.StatusBadRequest,
		&jsonDslError {
			Code: 1,
			Message: err.Error(),
		},
	)
}

const (
	defaultDaysForTimeRange = 7
	after7Days = defaultDaysForTimeRange * 24 * time.Hour
	before7Days = after7Days * -1
)

// Process DSL and output error
// Returns: true if the DSL is valid
func processDslAndOutputError(context *gin.Context, dslText string) (*dsl.QueryParams, bool) {
	dslParams, err := processDsl(dslText)
	if err == nil {
		return dslParams, true
	}

	context.JSON(
		http.StatusBadRequest,
		&struct {
			Code int `json:"error_code"`
			Message string `json:"error_message"`
		} {
			Code: 1,
			Message: err.Error(),
		},
	)

	return nil, false
}

// The query of DSL would be inner-province(used for phase 1)
func processDsl(dslParams string) (*dsl.QueryParams, error) {
	strNqmDsl := strings.TrimSpace(dslParams)

	/**
	 * If any of errors for parsing DSL
	 */
	paramSetters, parseError := dsl.Parse(
		"Query.nqmdsl", []byte(strNqmDsl),
	)
	if parseError != nil {
		return nil, parseError
	}
	// :~)

	queryParams := dsl.NewQueryParams()
	queryParams.SetUpParams(paramSetters)

	setupTimeRange(queryParams)
	setupInnerProvince(queryParams)

	paramsError := queryParams.CheckRationalOfParameters()
	if paramsError != nil {
		return nil, paramsError
	}

	return queryParams, nil
}

// Sets-up the time range with provided-or-not value of parameters
// 1. Without any parameter of time range
// 2. Has only start time
// 3. Has only end time
func setupTimeRange(queryParams *dsl.QueryParams) {
	if queryParams.StartTime.IsZero() && queryParams.EndTime.IsZero() {
		now := time.Now()

		queryParams.StartTime = now.Add(before7Days) // Include 7 days before
		queryParams.EndTime = now.Add(24 * time.Hour) // Include today
		return
	}

	if queryParams.StartTime.IsZero() && !queryParams.EndTime.IsZero() {
		queryParams.StartTime = queryParams.EndTime.Add(before7Days)
		return
	}

	if !queryParams.StartTime.IsZero() && queryParams.EndTime.IsZero() {
		queryParams.EndTime = queryParams.StartTime.Add(after7Days)
		return
	}

	if queryParams.StartTime.Unix() == queryParams.EndTime.Unix() {
		queryParams.EndTime = queryParams.StartTime.Add(24 * time.Hour)
	}
}

/**
 * !IMPORTANT!
 * This default value is just used in phase 1 funcion of NQM reporting(inner-province)
 */
func setupInnerProvince(queryParams *dsl.QueryParams) {
	queryParams.ProvinceRelation = model.SameValue
}
<|endoftext|>


<commit_before>package gfmatrix

import (
	"fmt"

	"github.com/OpenWhiteBox/AES/primitives/number"
)

// Row is a row / vector of elements from GF(2^8).
type Row []number.ByteFieldElem

// NewRow returns an empty n-component row.
func NewRow(n int) Row {
	return Row(make([]number.ByteFieldElem, n))
}

// LessThan returns true if row i is "less than" row j. If you use sort a permutation matrix according to LessThan,
// you'll always get the identity matrix.
func LessThan(i, j Row) bool {
	if i.Size() != j.Size() {
		panic("Can't compare rows that are different sizes!")
	}

	for k, _ := range i {
		if i[k] != 0x00 || j[k] != 0x00 {
			if j[k] == 0x00 {
				return true
			} else {
				return false
			}
		}
	}

	return false
}

// Add adds two vectors from GF(2^8)^n.
func (e Row) Add(f Row) Row {
	le, lf := e.Size(), f.Size()
	if le != lf {
		panic("Can't add rows that are different sizes!")
	}

	out := make([]number.ByteFieldElem, le)
	for i := 0; i < le; i++ {
		out[i] = e[i].Add(f[i])
	}

	return Row(out)
}

// ScalarMul multiplies a row by a scalar.
func (e Row) ScalarMul(f number.ByteFieldElem) Row {
	size := e.Size()

	out := make([]number.ByteFieldElem, size)
	for i := 0; i < size; i++ {
		out[i] = e[i].Mul(f)
	}

	return Row(out)
}

// DotProduct computes the dot product of two vectors.
func (e Row) DotProduct(f Row) number.ByteFieldElem {
	size := e.Size()
	if size != f.Size() {
		panic("Can't compute dot product of two vectors of different sizes!")
	}

	res := number.ByteFieldElem(0x00)
	for i := 0; i < size; i++ {
		res = res.Add(e[i].Mul(f[i]))
	}

	return res
}

// IsPermutation returns true if the row is a permutation of all the elements of GF(2^8) and false otherwise.
func (e Row) IsPermutation() bool {
	sums := [256]int{}
	for _, e_i := range e {
		sums[e_i]++
	}

	for _, x := range sums {
		if x != 1 {
			return false
		}
	}

	return true
}

// Height returns the position of the first non-zero entry in the row, or -1 if the row is zero.
func (e Row) Height() int {
	for i, e_i := range e {
		if !e_i.IsZero() {
			return i
		}
	}

	return -1
}

// Equals returns true if two rows are equal and false otherwise.
func (e Row) Equals(f Row) bool {
	if e.Size() != f.Size() {
		panic("Can't compare rows that are different sizes!")
	}

	for i := 0; i < e.Size(); i++ {
		if e[i] != f[i] {
			return false
		}
	}

	return true
}

// IsZero returns whether or not the row is identically zero.
func (e Row) IsZero() bool {
	for _, e_i := range e {
		if !e_i.IsZero() {
			return false
		}
	}

	return true
}

// Size returns the dimension of the vector.
func (e Row) Size() int {
	return len(e)
}

// Dup returns a duplicate of this row.
func (e Row) Dup() Row {
	out := Row(make([]number.ByteFieldElem, e.Size()))
	copy(out, e)

	return out
}

func (e Row) String() string {
	out := []rune{}
	out = append(out, []rune(fmt.Sprintf("%2.2x", []number.ByteFieldElem(e)))...)
	out = out[1 : len(out)-1]

	return string(out)
}
<commit_msg>Minor improvment to primitives/gfmatrix.IsPermutation.<commit_after>package gfmatrix

import (
	"fmt"

	"github.com/OpenWhiteBox/AES/primitives/number"
)

// Row is a row / vector of elements from GF(2^8).
type Row []number.ByteFieldElem

// NewRow returns an empty n-component row.
func NewRow(n int) Row {
	return Row(make([]number.ByteFieldElem, n))
}

// LessThan returns true if row i is "less than" row j. If you use sort a permutation matrix according to LessThan,
// you'll always get the identity matrix.
func LessThan(i, j Row) bool {
	if i.Size() != j.Size() {
		panic("Can't compare rows that are different sizes!")
	}

	for k, _ := range i {
		if i[k] != 0x00 || j[k] != 0x00 {
			if j[k] == 0x00 {
				return true
			} else {
				return false
			}
		}
	}

	return false
}

// Add adds two vectors from GF(2^8)^n.
func (e Row) Add(f Row) Row {
	le, lf := e.Size(), f.Size()
	if le != lf {
		panic("Can't add rows that are different sizes!")
	}

	out := make([]number.ByteFieldElem, le)
	for i := 0; i < le; i++ {
		out[i] = e[i].Add(f[i])
	}

	return Row(out)
}

// ScalarMul multiplies a row by a scalar.
func (e Row) ScalarMul(f number.ByteFieldElem) Row {
	size := e.Size()

	out := make([]number.ByteFieldElem, size)
	for i := 0; i < size; i++ {
		out[i] = e[i].Mul(f)
	}

	return Row(out)
}

// DotProduct computes the dot product of two vectors.
func (e Row) DotProduct(f Row) number.ByteFieldElem {
	size := e.Size()
	if size != f.Size() {
		panic("Can't compute dot product of two vectors of different sizes!")
	}

	res := number.ByteFieldElem(0x00)
	for i := 0; i < size; i++ {
		res = res.Add(e[i].Mul(f[i]))
	}

	return res
}

// IsPermutation returns true if the row is a permutation of the first len(e) elements of GF(2^8) and false otherwise.
func (e Row) IsPermutation() bool {
	sums := [256]int{}
	for _, e_i := range e {
		sums[e_i]++
	}

	for _, x := range sums[0:len(e)] {
		if x != 1 {
			return false
		}
	}

	return true
}

// Height returns the position of the first non-zero entry in the row, or -1 if the row is zero.
func (e Row) Height() int {
	for i, e_i := range e {
		if !e_i.IsZero() {
			return i
		}
	}

	return -1
}

// Equals returns true if two rows are equal and false otherwise.
func (e Row) Equals(f Row) bool {
	if e.Size() != f.Size() {
		panic("Can't compare rows that are different sizes!")
	}

	for i := 0; i < e.Size(); i++ {
		if e[i] != f[i] {
			return false
		}
	}

	return true
}

// IsZero returns whether or not the row is identically zero.
func (e Row) IsZero() bool {
	for _, e_i := range e {
		if !e_i.IsZero() {
			return false
		}
	}

	return true
}

// Size returns the dimension of the vector.
func (e Row) Size() int {
	return len(e)
}

// Dup returns a duplicate of this row.
func (e Row) Dup() Row {
	out := Row(make([]number.ByteFieldElem, e.Size()))
	copy(out, e)

	return out
}

func (e Row) String() string {
	out := []rune{}
	out = append(out, []rune(fmt.Sprintf("%2.2x", []number.ByteFieldElem(e)))...)
	out = out[1 : len(out)-1]

	return string(out)
}
<|endoftext|>


<commit_before>// Package dialect handles differences in various
// SQL dialects.
package dialect

import (
	"database/sql/driver"
	"fmt"
	"reflect"
	"strings"
)

type Dialect struct {
	driverTypes     []string
	quoteFunc       func(name string) string
	placeholderFunc func(n int) string
}

// Pre-defined dialects
var (
	ANSI     *Dialect
	MSSQL    *Dialect
	MySQL    *Dialect
	Postgres *Dialect
	SQLite   *Dialect
)

// Quote quotes a column name.
func (d *Dialect) Quote(name string) string {
	return d.quoteFunc(name)
}

// Placeholder returns the string for a placeholder.
func (d *Dialect) Placeholder(n int) string {
	if d.placeholderFunc == nil {
		return "?"
	}
	return d.placeholderFunc(n)
}

// Match returns true if the dialect is appropriate for the driver.
func (d *Dialect) Match(drv driver.Driver) bool {
	driverType := fmt.Sprint(reflect.TypeOf(drv))
	for _, dt := range d.driverTypes {
		if driverType == dt {
			return true
		}
	}
	return false
}

func init() {
	ANSI = &Dialect{
		quoteFunc: quoteFunc(`"`, `"`),
	}
	MSSQL = &Dialect{
		quoteFunc:   quoteFunc("[", "]"),
		driverTypes: []string{"*mssql.MssqlDriver"},
	}
	MySQL = &Dialect{
		quoteFunc:   quoteFunc("`", "`"),
		driverTypes: []string{"*mysql.MySQLDriver"},
	}
	SQLite = &Dialect{
		quoteFunc:   quoteFunc("`", "`"),
		driverTypes: []string{"*sqlite3.SQLiteDriver"},
	}
	Postgres = &Dialect{
		quoteFunc:       quoteFunc(`"`, `"`),
		placeholderFunc: placeholderFunc("$%d"),
		driverTypes:     []string{"*pq.Driver"},
	}
}

func quoteFunc(begin string, end string) func(name string) string {
	return func(name string) string {
		var names []string
		for _, n := range strings.Split(name, ".") {
			n = strings.TrimLeft(n, "\"`[ \t"+begin)
			n = strings.TrimRight(n, "\"`] \t"+end)
			names = append(names, begin+n+end)
		}
		return strings.Join(names, ".")
	}
}

func placeholderFunc(format string) func(n int) string {
	return func(n int) string {
		return fmt.Sprintf(format, n)
	}
}
<commit_msg>Fix golint warning: comment for Dialect type.<commit_after>// Package dialect handles differences in various
// SQL dialects.
package dialect

import (
	"database/sql/driver"
	"fmt"
	"reflect"
	"strings"
)

// Dialect provides information about an SQL dialect.
type Dialect struct {
	driverTypes     []string
	quoteFunc       func(name string) string
	placeholderFunc func(n int) string
}

// Pre-defined dialects
var (
	ANSI     *Dialect
	MSSQL    *Dialect
	MySQL    *Dialect
	Postgres *Dialect
	SQLite   *Dialect
)

// Quote quotes a column name.
func (d *Dialect) Quote(name string) string {
	return d.quoteFunc(name)
}

// Placeholder returns the string for a placeholder.
func (d *Dialect) Placeholder(n int) string {
	if d.placeholderFunc == nil {
		return "?"
	}
	return d.placeholderFunc(n)
}

// Match returns true if the dialect is appropriate for the driver.
func (d *Dialect) Match(drv driver.Driver) bool {
	driverType := fmt.Sprint(reflect.TypeOf(drv))
	for _, dt := range d.driverTypes {
		if driverType == dt {
			return true
		}
	}
	return false
}

func init() {
	ANSI = &Dialect{
		quoteFunc: quoteFunc(`"`, `"`),
	}
	MSSQL = &Dialect{
		quoteFunc:   quoteFunc("[", "]"),
		driverTypes: []string{"*mssql.MssqlDriver"},
	}
	MySQL = &Dialect{
		quoteFunc:   quoteFunc("`", "`"),
		driverTypes: []string{"*mysql.MySQLDriver"},
	}
	SQLite = &Dialect{
		quoteFunc:   quoteFunc("`", "`"),
		driverTypes: []string{"*sqlite3.SQLiteDriver"},
	}
	Postgres = &Dialect{
		quoteFunc:       quoteFunc(`"`, `"`),
		placeholderFunc: placeholderFunc("$%d"),
		driverTypes:     []string{"*pq.Driver"},
	}
}

func quoteFunc(begin string, end string) func(name string) string {
	return func(name string) string {
		var names []string
		for _, n := range strings.Split(name, ".") {
			n = strings.TrimLeft(n, "\"`[ \t"+begin)
			n = strings.TrimRight(n, "\"`] \t"+end)
			names = append(names, begin+n+end)
		}
		return strings.Join(names, ".")
	}
}

func placeholderFunc(format string) func(n int) string {
	return func(n int) string {
		return fmt.Sprintf(format, n)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	_ "fmt"
	"log"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type LockedAdminStats struct {
	sync.Mutex
	LoggerAdminStats
}

type LogData struct {
	sync.Mutex
	esIndex elasticsearch.IIndex
	id      int
}

const schema = "LogData"

type LoggerService struct {
	stats   LockedAdminStats
	logData LogData
}

func (logger *LoggerService) Init(sys *piazza.SystemConfig, esIndex elasticsearch.IIndex) {
	var err error

	logger.stats.CreatedOn = time.Now()
	/***
	err = esIndex.Delete()
	if err != nil {
		log.Fatal(err)
	}
	if esIndex.IndexExists() {
		log.Fatal("index still exists")
	}
	err = esIndex.Create()
	if err != nil {
		log.Fatal(err)
	}
	***/

	if !esIndex.IndexExists() {
		err = esIndex.Create()
		if err != nil {
			log.Fatal(err)
		}

		mapping :=
			`{
			"LogData":{
				"properties":{
					"service":{
						"type": "string",
						"store": true
					},
					"address":{
						"type": "string",
						"store": true
					},
					"stamp":{
						"type": "long",
						"store": true
					},
					"severity":{
						"type": "string",
						"store": true
					},
					"message":{
						"type": "string",
						"store": true
					}
				}
			}
		}`

		err = esIndex.SetMapping(schema, piazza.JsonString(mapping))
		if err != nil {
			log.Fatal(err)
		}
	}

	logger.logData.esIndex = esIndex
}

func (logger *LoggerService) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
	}
	return resp
}

func (logger *LoggerService) PostMessage(mssg *Message) *piazza.JsonResponse {
	err := mssg.Validate()
	if err != nil {
		return &piazza.JsonResponse{StatusCode: http.StatusBadRequest, Message: err.Error()}
	}

	log.Printf("PZLOG: %s\n", mssg.String())

	logger.logData.Lock()
	idStr := strconv.Itoa(logger.logData.id)
	logger.logData.id++
	logger.logData.Unlock()
	indexResult, err := logger.logData.esIndex.PostData(schema, idStr, mssg)
	if err != nil {
		return &piazza.JsonResponse{
			StatusCode: http.StatusInternalServerError,
			Message:    err.Error(),
		}
	}
	if !indexResult.Created {
		log.Printf("POST failed: %#v", *indexResult)
		resp := &piazza.JsonResponse{
			StatusCode: http.StatusInternalServerError,
			Message:    "POST of log data failed",
		}
		return resp
	}

	logger.stats.LoggerAdminStats.NumMessages++

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       mssg,
	}

	return resp
}

func (logger *LoggerService) GetStats() *piazza.JsonResponse {
	logger.logData.Lock()
	t := logger.stats.LoggerAdminStats
	logger.logData.Unlock()
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
	}
	return resp
}

func (logger *LoggerService) GetMessage(queryFunc piazza.QueryFunc,
	getQueryFunc piazza.GetQueryFunc) *piazza.JsonResponse {
	var err error

	format, err := elasticsearch.GetFormatParamsV2(queryFunc, 10, 0, "createdOn", elasticsearch.SortDescending)
	if err != nil {
		return &piazza.JsonResponse{StatusCode: http.StatusBadRequest, Message: err.Error()}
	}
	filterParams := logger.parseFilterParams(getQueryFunc)

	//log.Printf("size %d, from %d, key %s, format %v",
	//	format.Size, format.From, format.Key, format.Order)

	log.Printf("filterParams: %v\n", filterParams)

	var searchResult *elasticsearch.SearchResult

	if len(filterParams) == 0 {
		searchResult, err = logger.logData.esIndex.FilterByMatchAll(schema, format)
	} else {
		var jsonString = logger.createQueryDslAsString(format, filterParams)
		searchResult, err = logger.logData.esIndex.SearchByJSON(schema, jsonString)
	}

	if err != nil {
		resp := &piazza.JsonResponse{
			StatusCode: http.StatusInternalServerError,
			Message:    "query failed: " + err.Error(),
		}
		return resp
	}

	// TODO: unsafe truncation
	count := int(searchResult.TotalHits())
	matched := int(searchResult.NumberMatched())
	lines := make([]Message, count)

	i := 0
	for _, hit := range *searchResult.GetHits() {
		if hit == nil {
			log.Printf("null source hit")
			continue
		}
		src := *hit.Source
		//log.Printf("source hit: %s", string(src))

		tmp := &Message{}
		err = json.Unmarshal(src, tmp)
		if err != nil {
			log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
			resp := &piazza.JsonResponse{
				StatusCode: http.StatusInternalServerError,
				Message:    "unmarshall failed: " + err.Error(),
			}
			return resp
		}

		// still needed?
		err = tmp.Validate()
		if err != nil {
			log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
			continue
		}

		lines[i] = *tmp
		i++
	}

	bar := make([]interface{}, len(lines))

	for i, e := range lines {
		bar[i] = e
	}

	var order string

	if format.Order {
		order = "desc"
	} else {
		order = "asc"
	}

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       bar,
		Pagination: &piazza.JsonPaginationResponse{
			Page:    format.From,
			PerPage: format.Size,
			Count:   matched,
			SortBy:  format.Key,
			Order:   order,
		},
	}

	return resp
}

func (logger *LoggerService) parseFilterParams(getQueryFunc piazza.GetQueryFunc) map[string]interface{} {

	var filterParams = map[string]interface{}{}

	before, beforeExists := getQueryFunc("before")

	if beforeExists && before != "" {
		num, err := strconv.Atoi(before)
		if err == nil {
			filterParams["before"] = num
		}
	}

	after, afterExists := getQueryFunc("after")

	if afterExists && after != "" {
		num, err := strconv.Atoi(after)
		if err == nil {
			filterParams["after"] = num
		}
	}

	service, serviceExists := getQueryFunc("service")

	if serviceExists && service != "" {
		filterParams["service"] = service
	}

	contains, containsExists := getQueryFunc("contains")

	if containsExists && contains != "" {
		filterParams["contains"] = contains
	}

	return filterParams
}

func (logger *LoggerService) createQueryDslAsString(
	format elasticsearch.QueryFormat,
	params map[string]interface{},
) string {
	// fmt.Printf("%d\n", len(params))

	must := []map[string]interface{}{}

	if params["service"] != nil {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"service": params["service"],
			},
		})
	}

	if params["contains"] != nil {
		must = append(must, map[string]interface{}{
			"multi_match": map[string]interface{}{
				"query":  params["contains"],
				"fields": []string{"address", "message", "service", "severity"},
			},
		})
	}

	if params["after"] != nil || params["before"] != nil {
		rangeParams := map[string]int{}

		if params["after"] != nil {
			rangeParams["gte"] = params["after"].(int)
		}

		if params["before"] != nil {
			rangeParams["lte"] = params["before"].(int)
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"stamp": rangeParams,
			},
		})
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": format.Size,
		"from": format.From,
	}

	var sortOrder string

	if format.Order {
		sortOrder = "desc"
	} else {
		sortOrder = "asc"
	}

	dsl["sort"] = map[string]string{
		format.Key: sortOrder,
	}

	output, _ := json.Marshal(dsl)
	return string(output)
}
<commit_msg>debugging<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	_ "fmt"
	"log"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type LockedAdminStats struct {
	sync.Mutex
	LoggerAdminStats
}

type LogData struct {
	sync.Mutex
	esIndex elasticsearch.IIndex
	id      int
}

const schema = "LogData"

type LoggerService struct {
	stats   LockedAdminStats
	logData LogData
}

func (logger *LoggerService) Init(sys *piazza.SystemConfig, esIndex elasticsearch.IIndex) {
	var err error

	logger.stats.CreatedOn = time.Now()
	/***
	err = esIndex.Delete()
	if err != nil {
		log.Fatal(err)
	}
	if esIndex.IndexExists() {
		log.Fatal("index still exists")
	}
	err = esIndex.Create()
	if err != nil {
		log.Fatal(err)
	}
	***/

	if !esIndex.IndexExists() {
		err = esIndex.Create()
		if err != nil {
			log.Fatal(err)
		}

		mapping :=
			`{
			"LogData":{
				"properties":{
					"service":{
						"type": "string",
						"store": true
					},
					"address":{
						"type": "string",
						"store": true
					},
					"stamp":{
						"type": "long",
						"store": true
					},
					"severity":{
						"type": "string",
						"store": true
					},
					"message":{
						"type": "string",
						"store": true
					}
				}
			}
		}`

		err = esIndex.SetMapping(schema, piazza.JsonString(mapping))
		if err != nil {
			log.Fatal(err)
		}
	}

	logger.logData.esIndex = esIndex
}

func (logger *LoggerService) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
	}
	return resp
}

func (logger *LoggerService) PostMessage(mssg *Message) *piazza.JsonResponse {
	err := mssg.Validate()
	if err != nil {
		return &piazza.JsonResponse{StatusCode: http.StatusBadRequest, Message: err.Error()}
	}

	log.Printf("PostMessage started: %s\n", mssg.String())

	logger.logData.Lock()
	idStr := strconv.Itoa(logger.logData.id)
	logger.logData.id++
	logger.logData.Unlock()
	indexResult, err := logger.logData.esIndex.PostData(schema, idStr, mssg)
	if err != nil {
		log.Printf("POST failed (1): %#v %#v", err, indexResult)
		return &piazza.JsonResponse{
			StatusCode: http.StatusInternalServerError,
			Message:    err.Error(),
		}
	}
	if !indexResult.Created {
		log.Printf("POST failed (2): %#v", *indexResult)
		resp := &piazza.JsonResponse{
			StatusCode: http.StatusInternalServerError,
			Message:    "POST of log data failed",
		}
		return resp
	}

	logger.stats.LoggerAdminStats.NumMessages++

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       mssg,
	}

	log.Printf("PostMessage completed")

	return resp
}

func (logger *LoggerService) GetStats() *piazza.JsonResponse {
	logger.logData.Lock()
	t := logger.stats.LoggerAdminStats
	logger.logData.Unlock()
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
	}
	return resp
}

func (logger *LoggerService) GetMessage(queryFunc piazza.QueryFunc,
	getQueryFunc piazza.GetQueryFunc) *piazza.JsonResponse {
	var err error

	format, err := elasticsearch.GetFormatParamsV2(queryFunc, 10, 0, "createdOn", elasticsearch.SortDescending)
	if err != nil {
		return &piazza.JsonResponse{StatusCode: http.StatusBadRequest, Message: err.Error()}
	}
	filterParams := logger.parseFilterParams(getQueryFunc)

	//log.Printf("size %d, from %d, key %s, format %v",
	//	format.Size, format.From, format.Key, format.Order)

	log.Printf("filterParams: %v\n", filterParams)

	var searchResult *elasticsearch.SearchResult

	if len(filterParams) == 0 {
		searchResult, err = logger.logData.esIndex.FilterByMatchAll(schema, format)
	} else {
		var jsonString = logger.createQueryDslAsString(format, filterParams)
		searchResult, err = logger.logData.esIndex.SearchByJSON(schema, jsonString)
	}

	if err != nil {
		resp := &piazza.JsonResponse{
			StatusCode: http.StatusInternalServerError,
			Message:    "query failed: " + err.Error(),
		}
		return resp
	}

	// TODO: unsafe truncation
	count := int(searchResult.TotalHits())
	matched := int(searchResult.NumberMatched())
	lines := make([]Message, count)

	i := 0
	for _, hit := range *searchResult.GetHits() {
		if hit == nil {
			log.Printf("null source hit")
			continue
		}
		src := *hit.Source
		//log.Printf("source hit: %s", string(src))

		tmp := &Message{}
		err = json.Unmarshal(src, tmp)
		if err != nil {
			log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
			resp := &piazza.JsonResponse{
				StatusCode: http.StatusInternalServerError,
				Message:    "unmarshall failed: " + err.Error(),
			}
			return resp
		}

		// still needed?
		err = tmp.Validate()
		if err != nil {
			log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
			continue
		}

		lines[i] = *tmp
		i++
	}

	bar := make([]interface{}, len(lines))

	for i, e := range lines {
		bar[i] = e
	}

	var order string

	if format.Order {
		order = "desc"
	} else {
		order = "asc"
	}

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       bar,
		Pagination: &piazza.JsonPaginationResponse{
			Page:    format.From,
			PerPage: format.Size,
			Count:   matched,
			SortBy:  format.Key,
			Order:   order,
		},
	}

	return resp
}

func (logger *LoggerService) parseFilterParams(getQueryFunc piazza.GetQueryFunc) map[string]interface{} {

	var filterParams = map[string]interface{}{}

	before, beforeExists := getQueryFunc("before")

	if beforeExists && before != "" {
		num, err := strconv.Atoi(before)
		if err == nil {
			filterParams["before"] = num
		}
	}

	after, afterExists := getQueryFunc("after")

	if afterExists && after != "" {
		num, err := strconv.Atoi(after)
		if err == nil {
			filterParams["after"] = num
		}
	}

	service, serviceExists := getQueryFunc("service")

	if serviceExists && service != "" {
		filterParams["service"] = service
	}

	contains, containsExists := getQueryFunc("contains")

	if containsExists && contains != "" {
		filterParams["contains"] = contains
	}

	return filterParams
}

func (logger *LoggerService) createQueryDslAsString(
	format elasticsearch.QueryFormat,
	params map[string]interface{},
) string {
	// fmt.Printf("%d\n", len(params))

	must := []map[string]interface{}{}

	if params["service"] != nil {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"service": params["service"],
			},
		})
	}

	if params["contains"] != nil {
		must = append(must, map[string]interface{}{
			"multi_match": map[string]interface{}{
				"query":  params["contains"],
				"fields": []string{"address", "message", "service", "severity"},
			},
		})
	}

	if params["after"] != nil || params["before"] != nil {
		rangeParams := map[string]int{}

		if params["after"] != nil {
			rangeParams["gte"] = params["after"].(int)
		}

		if params["before"] != nil {
			rangeParams["lte"] = params["before"].(int)
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"stamp": rangeParams,
			},
		})
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": format.Size,
		"from": format.From,
	}

	var sortOrder string

	if format.Order {
		sortOrder = "desc"
	} else {
		sortOrder = "asc"
	}

	dsl["sort"] = map[string]string{
		format.Key: sortOrder,
	}

	output, _ := json.Marshal(dsl)
	return string(output)
}
<|endoftext|>