<commit_before>package ogdatv21

import (
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"net/url"
	"strconv"
	"time"
)

const OGDTimeSpecifier = "2006-01-02T15:04:05" // RFC 3339 = ISO 8601 ohne Zeitzone
const (
	OGDTime2 = time.RFC3339Nano
	OGDTime3 = time.RFC3339
	OGDTime1 = OGDTimeSpecifier
	OGDTimeUnknow
)

type Kategorie struct {
	NumID       int `json:"-"`
	ID          string
	PrettyName  string `json:"-"`
	RDFProperty string `json:"-"`
}

func (kat *Kategorie) String() string {
	return kat.PrettyName
}

var (
	Arbeit           = Kategorie{NumID: 1, ID: "arbeit", PrettyName: "Arbeit", RDFProperty: ""}
	Bevoelkerung     = Kategorie{NumID: 2, ID: "bevölkerung", PrettyName: "Bevölkerung", RDFProperty: ""}
	BildungForschung = Kategorie{NumID: 3, ID: "bildung-und-forschung", PrettyName: "Bildung und Forschung", RDFProperty: ""}
	FinanzRW         = Kategorie{NumID: 4, ID: "finanzen-und-rechnungswesen", PrettyName: "Finanzen und Rechnungswesen", RDFProperty: ""}
	GeographPlanung  = Kategorie{NumID: 5, ID: "geographie-und-planung", PrettyName: "Geographie und Planung", RDFProperty: ""}
	GesellSoziales   = Kategorie{NumID: 6, ID: "gesellschaft-und-soziales", PrettyName: "Gesellschaft und Soziales", RDFProperty: ""}
	Gesundheit       = Kategorie{NumID: 7, ID: "gesundheit", PrettyName: "Gesundheit", RDFProperty: ""}
	KunstKultur      = Kategorie{NumID: 8, ID: "kunst-und-kultur", PrettyName: "Kunst und Kultur", RDFProperty: ""}
	LandFW           = Kategorie{NumID: 9, ID: "land-und-forstwirtschaft", PrettyName: "Land und Forstwirtschaft", RDFProperty: ""}
	SportFZ          = Kategorie{NumID: 10, ID: "sport-und-freizeit", PrettyName: "Sport und Freizeit", RDFProperty: ""}
	Umwelt           = Kategorie{NumID: 11, ID: "umwelt", PrettyName: "Umwelt", RDFProperty: ""}
	VerkehrTechnik   = Kategorie{NumID: 12, ID: "verkehr-und-technik", PrettyName: "Verkehr und Technik", RDFProperty: ""}
	VerwaltPol       = Kategorie{NumID: 13, ID: "verwaltung-und-politik", PrettyName: "Verwaltung und Politik", RDFProperty: ""}
	WirtTourism      = Kategorie{NumID: 14, ID: "wirtschaft-und-tourismus", PrettyName: "Wirtschaft und Tourismus", RDFProperty: ""}
)

var categories = []Kategorie{
	Arbeit,
	Bevoelkerung,
	BildungForschung,
	FinanzRW,
	GeographPlanung,
	GesellSoziales,
	Gesundheit,
	KunstKultur,
	LandFW,
	SportFZ,
	Umwelt,
	VerkehrTechnik,
	VerwaltPol,
	WirtTourism,
}

var categorymap = make(map[string]Kategorie)

type Tags string
type ResourceSpecifier string

type Cycle struct {
	NumID                       int
	DomainCode                  string
	MD_MaintenanceFrequencyCode string
	Name_DE                     string
}

var (
	CycCont     = Cycle{1, "001", "continual", "kontinuierlich"}
	CycDaily    = Cycle{2, "002", "daily", "täglich"}
	CycWeekly   = Cycle{3, "003", "weekly", "wöchentlich"}
	CycFortNly  = Cycle{4, "004", "fortnightly", "14-tägig"}
	CycMonthly  = Cycle{5, "005", "monthly", "monatlich"}
	CycQuart    = Cycle{6, "006", "quarterly", "quartalsweise"}
	CycBiAnn    = Cycle{7, "007", "biannually", "halbjährlich"}
	CycAnnually = Cycle{8, "008", "annually", "jährlich"}
	CycNeeded   = Cycle{9, "009", "asNeeded", "nach Bedarf"}
	CycIrreg    = Cycle{10, "010", "irregular", "unregelmäßig"}
	CycNP       = Cycle{11, "011", "notPlanned", "nicht geplant"}
	CycUnknown  = Cycle{12, "012", "unknown", "unbekannt"}
)

var cycles = []Cycle{
	CycCont,
	CycDaily,
	CycWeekly,
	CycFortNly,
	CycMonthly,
	CycQuart,
	CycBiAnn,
	CycAnnually,
	CycNeeded,
	CycIrreg,
	CycNP,
	CycUnknown,
}

type Url struct {
	*url.URL
	Raw string
}

type Identfier struct {
	*uuid.UUID
	Raw string
}

func (id *Identfier) String() string {
	return id.Raw
}

type Time struct {
	*time.Time
	Raw    string
	Format string
}

func (time *Time) String() string {
	return time.Raw
}

func (cyc *Cycle) String() string {
	return cyc.Name_DE
}

func cmpstrtocycle(raw string, cyc Cycle) bool {
	if raw == cyc.Name_DE || raw == cyc.DomainCode || raw == cyc.MD_MaintenanceFrequencyCode {
		return true
	}
	if len(raw) > 0 {
		if i, err := strconv.Atoi(raw); err == nil && i == cyc.NumID {
			return true
		}
	}
	return false
}

func (cyc *Cycle) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	var found bool
	var idx int
	var matchcyc Cycle

	for idx, matchcyc = range cycles {
		if found := cmpstrtocycle(raw, matchcyc); found == true {
			break
		}
	}

	if found {
		*cyc = cycles[idx]
	} else {
		cyc.NumID = -1
		cyc.Name_DE = "**** NON cycle spec **** - " + raw
		cyc.MD_MaintenanceFrequencyCode = cyc.Name_DE
	}
	return nil
}

func (ogdtime *Time) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}
	ogdtime.Raw = raw

	ogdtime.Format = OGDTime1
	t, err := time.Parse(ogdtime.Format, raw)
	if err != nil {
		ogdtime.Format = OGDTime2
		t, err = time.Parse(ogdtime.Format, raw)
		if err != nil {
			ogdtime.Format = OGDTime3
			t, err = time.Parse(ogdtime.Format, raw)
			if err != nil {
				ogdtime.Format = OGDTimeUnknow
			}
		}
	}
	ogdtime.Time = &t
	return nil
}

func (u *Url) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}
	u.Raw = raw
	url, _ := url.Parse(raw) // an actuall error is not important. If url can not be parsed, result will be nil, which is fine here
	u.URL = url
	return nil
}

func (id *Identfier) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	id.Raw = string(raw)
	if uuid := uuid.Parse(raw); uuid != nil {
		id.UUID = &uuid
	}
	return nil
}

func (kat *Kategorie) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	corecat, found := categorymap[raw]
	if !found {
		kat.NumID = -1
		kat.ID = raw
		kat.PrettyName = "**** NON core category **** - " + kat.ID
	} else {
		*kat = corecat
	}
	return nil
}

type Extras struct {
	// Core
	Metadata_Identifier Identfier   `json:"metadata_identifier"` // CKAN uses since API Version 2 a UUID V4, cf. https://github.com/okfn/ckan/blob/master/ckan/model/types.py
	Metadata_Modified   *Time       `json:"metadata_modified"`
	Categorization      []Kategorie `json:"categorization"`
	Begin_DateTime      *Time       `json:"begin_datetime"`

	// Optional
	Schema_Name           *string `json:"schema_name"`
	Schema_Language       *string `json:"schema_language"`     // always "ger"
	Schema_Characterset   *string `json:"schema_characterset"` // always "utf8", cf. https://www.ghrsst.org/files/download.php?m=documents&f=ISO%2019115%20.pdf
	MetaData_Linkage      []Url   `json:"metadata_linkage"`
	Attribute_Description *string `json:"attribute_description"`
	Maintainer_Link       *Url    `json:"maintainer_link"`
	Publisher             *string `json:"publisher"`
	Geographich_Toponym   *string `json:"geographic_toponym"`

	/*  ON/EN/ISO 19115:2003: westBL (344) & eastBL (345) & southBL (346) & northBL (347)
	 * TODO: Specifiaction says a WKT of POLYGON should be used, which would make a
	 * POLYGON ((-180.00 -90.00, 180.00 90.00)) but Example states
	 * POLYGON (-180.00 -90.00, 180.00 90.00)
	 * UNDER CLARIFICATION
	 */
	Geographic_BBox  *string `json:"geographic_bbox"`
	End_DateTime     *Time   `json:"end_datetime"`
	Update_Frequency *Cycle  `json:"update_frequency"`
	Lineage_Quality  *string `json:"lineage_quality"`
	EnTitleDesc      *string `json:"en_title_and_desc"`
}

type Resource struct {
	// Core
	URL    *Url              `json:"url"`
	Format ResourceSpecifier `json:"format"`

	// Optional
	Name         *string `json:"name"`
	Created      *Time   `json:"created"`
	LastModified *Time   `json:"last_modified"`

	/*
	 * dcat:bytes a rdf:Property, owl:DatatypeProperty;
	 * rdfs:isDefinedBy <http://www.w3.org/ns/dcat>;
	 * rdfs:label "size in bytes";
	 * rdfs:comment "describe size of resource in bytes";
	 * rdfs:domain dcat:Distribution;
	 * rdfs:range xsd:integer .
	 */
	Size             *string `json:"size"`
	License_Citation *string `json:"license_citation"`
	Language         *string `json:"language"`
	/* Here we have a problem in spec 2.1. which says "nach ISO\IEC 10646-1", which means utf-8, utf-16 and utf-32.
	 * We would certainly support more encodings, as eg.
	 * ISO 19115 / B.5.10 MD_CharacterSetCode<> or
	 * http://www.iana.org/assignments/character-sets/character-sets.xml
	 */
	Encoding *string `json:"characterset"`
}

type MetaData struct {
	// Core
	Title       string `json:"title"`
	Description string `json:"notes"`
	Schlagworte []Tags `json:"tags"`
	Maintainer  string `json:"maintainer"`
	License     string `json:"license"` // Sollte URI des Lizenzdokuments sein

	// nested structs
	Extras   `json:"extras"`
	Resource []Resource `json:"resources"`
}

func init() {
	for idx, val := range categories {
		categorymap[val.ID] = categories[idx]
	}
}
<commit_msg>read OGD AT specification from file at init time<commit_after>package ogdatv21

import (
	"code.google.com/p/go-uuid/uuid"
	"encoding/csv"
	"encoding/json"
	"github.com/the42/ogdat"
	"io"
	"log"
	"net/url"
	"os"
	"strconv"
	"time"
)

const OGDTimeSpecifier = "2006-01-02T15:04:05" // RFC 3339 = ISO 8601 ohne Zeitzone
const ogdatv21specfile = "ogdat_spec-2.1.csv"
const (
	OGDTime2 = time.RFC3339Nano
	OGDTime3 = time.RFC3339
	OGDTime1 = OGDTimeSpecifier
	OGDTimeUnknow
)

var specmap map[int]*ogdat.Beschreibung

type Kategorie struct {
	NumID       int `json:"-"`
	ID          string
	PrettyName  string `json:"-"`
	RDFProperty string `json:"-"`
}

func (kat *Kategorie) String() string {
	return kat.PrettyName
}

var (
	Arbeit           = Kategorie{NumID: 1, ID: "arbeit", PrettyName: "Arbeit", RDFProperty: ""}
	Bevoelkerung     = Kategorie{NumID: 2, ID: "bevölkerung", PrettyName: "Bevölkerung", RDFProperty: ""}
	BildungForschung = Kategorie{NumID: 3, ID: "bildung-und-forschung", PrettyName: "Bildung und Forschung", RDFProperty: ""}
	FinanzRW         = Kategorie{NumID: 4, ID: "finanzen-und-rechnungswesen", PrettyName: "Finanzen und Rechnungswesen", RDFProperty: ""}
	GeographPlanung  = Kategorie{NumID: 5, ID: "geographie-und-planung", PrettyName: "Geographie und Planung", RDFProperty: ""}
	GesellSoziales   = Kategorie{NumID: 6, ID: "gesellschaft-und-soziales", PrettyName: "Gesellschaft und Soziales", RDFProperty: ""}
	Gesundheit       = Kategorie{NumID: 7, ID: "gesundheit", PrettyName: "Gesundheit", RDFProperty: ""}
	KunstKultur      = Kategorie{NumID: 8, ID: "kunst-und-kultur", PrettyName: "Kunst und Kultur", RDFProperty: ""}
	LandFW           = Kategorie{NumID: 9, ID: "land-und-forstwirtschaft", PrettyName: "Land und Forstwirtschaft", RDFProperty: ""}
	SportFZ          = Kategorie{NumID: 10, ID: "sport-und-freizeit", PrettyName: "Sport und Freizeit", RDFProperty: ""}
	Umwelt           = Kategorie{NumID: 11, ID: "umwelt", PrettyName: "Umwelt", RDFProperty: ""}
	VerkehrTechnik   = Kategorie{NumID: 12, ID: "verkehr-und-technik", PrettyName: "Verkehr und Technik", RDFProperty: ""}
	VerwaltPol       = Kategorie{NumID: 13, ID: "verwaltung-und-politik", PrettyName: "Verwaltung und Politik", RDFProperty: ""}
	WirtTourism      = Kategorie{NumID: 14, ID: "wirtschaft-und-tourismus", PrettyName: "Wirtschaft und Tourismus", RDFProperty: ""}
)

var categories = []Kategorie{
	Arbeit,
	Bevoelkerung,
	BildungForschung,
	FinanzRW,
	GeographPlanung,
	GesellSoziales,
	Gesundheit,
	KunstKultur,
	LandFW,
	SportFZ,
	Umwelt,
	VerkehrTechnik,
	VerwaltPol,
	WirtTourism,
}

var categorymap = make(map[string]Kategorie)

type Tags string
type ResourceSpecifier string

type Cycle struct {
	NumID                       int
	DomainCode                  string
	MD_MaintenanceFrequencyCode string
	Name_DE                     string
}

var (
	CycCont     = Cycle{1, "001", "continual", "kontinuierlich"}
	CycDaily    = Cycle{2, "002", "daily", "täglich"}
	CycWeekly   = Cycle{3, "003", "weekly", "wöchentlich"}
	CycFortNly  = Cycle{4, "004", "fortnightly", "14-tägig"}
	CycMonthly  = Cycle{5, "005", "monthly", "monatlich"}
	CycQuart    = Cycle{6, "006", "quarterly", "quartalsweise"}
	CycBiAnn    = Cycle{7, "007", "biannually", "halbjährlich"}
	CycAnnually = Cycle{8, "008", "annually", "jährlich"}
	CycNeeded   = Cycle{9, "009", "asNeeded", "nach Bedarf"}
	CycIrreg    = Cycle{10, "010", "irregular", "unregelmäßig"}
	CycNP       = Cycle{11, "011", "notPlanned", "nicht geplant"}
	CycUnknown  = Cycle{12, "012", "unknown", "unbekannt"}
)

var cycles = []Cycle{
	CycCont,
	CycDaily,
	CycWeekly,
	CycFortNly,
	CycMonthly,
	CycQuart,
	CycBiAnn,
	CycAnnually,
	CycNeeded,
	CycIrreg,
	CycNP,
	CycUnknown,
}

type Url struct {
	*url.URL
	Raw string
}

type Identfier struct {
	*uuid.UUID
	Raw string
}

func (id *Identfier) String() string {
	return id.Raw
}

type Time struct {
	*time.Time
	Raw    string
	Format string
}

func (time *Time) String() string {
	return time.Raw
}

func (cyc *Cycle) String() string {
	return cyc.Name_DE
}

func cmpstrtocycle(raw string, cyc Cycle) bool {
	if raw == cyc.Name_DE || raw == cyc.DomainCode || raw == cyc.MD_MaintenanceFrequencyCode {
		return true
	}
	if len(raw) > 0 {
		if i, err := strconv.Atoi(raw); err == nil && i == cyc.NumID {
			return true
		}
	}
	return false
}

func (cyc *Cycle) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	var found bool
	var idx int
	var matchcyc Cycle

	for idx, matchcyc = range cycles {
		if found := cmpstrtocycle(raw, matchcyc); found == true {
			break
		}
	}

	if found {
		*cyc = cycles[idx]
	} else {
		cyc.NumID = -1
		cyc.Name_DE = "**** NON cycle spec **** - " + raw
		cyc.MD_MaintenanceFrequencyCode = cyc.Name_DE
	}
	return nil
}

func (ogdtime *Time) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}
	ogdtime.Raw = raw

	ogdtime.Format = OGDTime1
	t, err := time.Parse(ogdtime.Format, raw)
	if err != nil {
		ogdtime.Format = OGDTime2
		t, err = time.Parse(ogdtime.Format, raw)
		if err != nil {
			ogdtime.Format = OGDTime3
			t, err = time.Parse(ogdtime.Format, raw)
			if err != nil {
				ogdtime.Format = OGDTimeUnknow
			}
		}
	}
	ogdtime.Time = &t
	return nil
}

func (u *Url) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}
	u.Raw = raw
	url, _ := url.Parse(raw) // an actuall error is not important. If url can not be parsed, result will be nil, which is fine here
	u.URL = url
	return nil
}

func (id *Identfier) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	id.Raw = string(raw)
	if uuid := uuid.Parse(raw); uuid != nil {
		id.UUID = &uuid
	}
	return nil
}

func (kat *Kategorie) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	corecat, found := categorymap[raw]
	if !found {
		kat.NumID = -1
		kat.ID = raw
		kat.PrettyName = "**** NON core category **** - " + kat.ID
	} else {
		*kat = corecat
	}
	return nil
}

type Extras struct {
	// Core
	Metadata_Identifier Identfier   `json:"metadata_identifier"` // CKAN uses since API Version 2 a UUID V4, cf. https://github.com/okfn/ckan/blob/master/ckan/model/types.py
	Metadata_Modified   *Time       `json:"metadata_modified"`
	Categorization      []Kategorie `json:"categorization"`
	Begin_DateTime      *Time       `json:"begin_datetime"`

	// Optional
	Schema_Name           *string `json:"schema_name"`
	Schema_Language       *string `json:"schema_language"`     // always "ger"
	Schema_Characterset   *string `json:"schema_characterset"` // always "utf8", cf. https://www.ghrsst.org/files/download.php?m=documents&f=ISO%2019115%20.pdf
	MetaData_Linkage      []Url   `json:"metadata_linkage"`
	Attribute_Description *string `json:"attribute_description"`
	Maintainer_Link       *Url    `json:"maintainer_link"`
	Publisher             *string `json:"publisher"`
	Geographich_Toponym   *string `json:"geographic_toponym"`

	/*  ON/EN/ISO 19115:2003: westBL (344) & eastBL (345) & southBL (346) & northBL (347)
	 * TODO: Specifiaction says a WKT of POLYGON should be used, which would make a
	 * POLYGON ((-180.00 -90.00, 180.00 90.00)) but Example states
	 * POLYGON (-180.00 -90.00, 180.00 90.00)
	 * UNDER CLARIFICATION
	 */
	Geographic_BBox  *string `json:"geographic_bbox"`
	End_DateTime     *Time   `json:"end_datetime"`
	Update_Frequency *Cycle  `json:"update_frequency"`
	Lineage_Quality  *string `json:"lineage_quality"`
	EnTitleDesc      *string `json:"en_title_and_desc"`
}

type Resource struct {
	// Core
	URL    *Url              `json:"url"`
	Format ResourceSpecifier `json:"format"`

	// Optional
	Name         *string `json:"name"`
	Created      *Time   `json:"created"`
	LastModified *Time   `json:"last_modified"`

	/*
	 * dcat:bytes a rdf:Property, owl:DatatypeProperty;
	 * rdfs:isDefinedBy <http://www.w3.org/ns/dcat>;
	 * rdfs:label "size in bytes";
	 * rdfs:comment "describe size of resource in bytes";
	 * rdfs:domain dcat:Distribution;
	 * rdfs:range xsd:integer .
	 */
	Size             *string `json:"size"`
	License_Citation *string `json:"license_citation"`
	Language         *string `json:"language"`
	/* Here we have a problem in spec 2.1. which says "nach ISO\IEC 10646-1", which means utf-8, utf-16 and utf-32.
	 * We would certainly support more encodings, as eg.
	 * ISO 19115 / B.5.10 MD_CharacterSetCode<> or
	 * http://www.iana.org/assignments/character-sets/character-sets.xml
	 */
	Encoding *string `json:"characterset"`
}

type MetaData struct {
	// Core
	Title       string `json:"title"`
	Description string `json:"notes"`
	Schlagworte []Tags `json:"tags"`
	Maintainer  string `json:"maintainer"`
	License     string `json:"license"` // Sollte URI des Lizenzdokuments sein

	// nested structs
	Extras   `json:"extras"`
	Resource []Resource `json:"resources"`
}

func loadogdatv21spec(filename string) (specmap map[int]*ogdat.Beschreibung) {
	reader, err := os.Open(filename)
	defer reader.Close()

	if err == nil {
		specmap = make(map[int]*ogdat.Beschreibung)
		csvreader := csv.NewReader(reader)
		csvreader.Comma = '|'

		for record, err := csvreader.Read(); err != io.EOF; record, err = csvreader.Read() {
			if len(record) < 12 {
				return nil
			}
			id, err := strconv.Atoi(record[0])
			if err != nil {
				return nil
			}
			var occ ogdat.Occurrence
			switch record[12][0] {
			case 'R':
				occ = ogdat.OccRequired
			case 'O':
				occ = ogdat.OccOptional
			}
			descrecord := ogdat.NewBeschreibung(id, occ, ogdat.Version21)
			descrecord.ID = id
			descrecord.Bezeichner = record[1]
			descrecord.OGD_Kurzname = record[2]
			descrecord.CKAN_Feld = record[3]
			descrecord.Anzahl = byte(record[4][0])
			descrecord.Definition_DE = record[5]
			descrecord.Erlauterung = record[6]
			descrecord.Beispiel = record[7]
			descrecord.ONA2270 = record[8]
			descrecord.ISO19115 = record[9]
			descrecord.RDFProperty = record[10]
			descrecord.Definition_EN = record[11]

			specmap[id] = descrecord
		}
		log.Printf("Info: Read %d %s specifiaction records", len(specmap), ogdat.Version21)
	} else {
		log.Printf("Warning: Can not read %s specification records", len(isolangfilemap))
	}
	return
}

func init() {
	for idx, val := range categories {
		categorymap[val.ID] = categories[idx]
	}

}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pbutil

import "log"

type Marshaler interface {
	Marshal() (data []byte, err error)
}

type Unmarshaler interface {
	Unmarshal(data []byte) error
}

func MustMarshal(m Marshaler) []byte {
	d, err := m.Marshal()
	if err != nil {
		log.Panicf("marshal protobuf type should never fail: %v", err)
	}
	return d
}

func MustUnmarshal(um Unmarshaler, data []byte) {
	if err := um.Unmarshal(data); err != nil {
		log.Panicf("unmarshal protobuf type should never fail: %v", err)
	}
}

func GetBool(v *bool) (vv bool, set bool) {
	if v == nil {
		return false, false
	}
	return *v, true
}

func Boolp(b bool) *bool { return &b }
<commit_msg>pkg/pbutil: use leveled log<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pbutil

import "github.com/coreos/pkg/capnslog"

var (
	plog = capnslog.NewPackageLogger("github.com/coreos/etcd/pkg", "flags")
)

type Marshaler interface {
	Marshal() (data []byte, err error)
}

type Unmarshaler interface {
	Unmarshal(data []byte) error
}

func MustMarshal(m Marshaler) []byte {
	d, err := m.Marshal()
	if err != nil {
		plog.Panicf("marshal should never fail (%v)", err)
	}
	return d
}

func MustUnmarshal(um Unmarshaler, data []byte) {
	if err := um.Unmarshal(data); err != nil {
		plog.Panicf("unmarshal should never fail (%v)", err)
	}
}

func GetBool(v *bool) (vv bool, set bool) {
	if v == nil {
		return false, false
	}
	return *v, true
}

func Boolp(b bool) *bool { return &b }
<|endoftext|>


<commit_before><commit_msg>Skip local rules when sending a sharing request<commit_after><|endoftext|>


<commit_before><commit_msg>enable/disable for analytics_pipeline as a whole<commit_after><|endoftext|>


<commit_before>package timeline

import (
	"log"

	"github.com/syndtr/goleveldb/leveldb"
)

type TimelineService struct {
	SlackClient         slackClient
	TimelineChannelID   string
	BlackListChannelIDs []string
	db                  leveldb.DB
	logger              log.Logger
}

func NewTimelineService(slackAPIToken, timelineChannelID string, blackListChannelIDs []string, db leveldb.DB, logger log.Logger) TimelineService {
	return TimelineService{
		SlackClient:         slackClient{Token: slackAPIToken},
		TimelineChannelID:   timelineChannelID,
		BlackListChannelIDs: blackListChannelIDs,
		db:                  db,
		logger:              logger,
	}
}

func (service *TimelineService) Run() error {
	messageChan := make(chan *slackMessage)
	errorChan := make(chan error)

	go service.SlackClient.polling(messageChan, errorChan)
	for {
		select {
		case msg := <-messageChan:
			if !service.isTargetMessage(msg) {
				service.logger.Println(msg)
				continue
			}
			key := msg.ChannelID + "-" + msg.TimeStamp
			_, err := service.db.Get([]byte(key), nil)
			if err == nil {
				continue
			}
			e := service.postMessage(msg)
			if e != nil {
				service.db.Put([]byte(key), []byte(key), nil)
				return e
			}
		case e := <-errorChan:
			return e
		default:
			break
		}
	}
	return nil
}

func (service *TimelineService) isTargetMessage(m *slackMessage) bool {
	return m.Type == "message" &&
		m.ChannelID != service.TimelineChannelID &&
		isPublic(m.ChannelID) &&
		!contains(service.BlackListChannelIDs, m.ChannelID)
}

func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

func (service *TimelineService) postMessage(m *slackMessage) error {
	// TODO cache?
	u, e := service.SlackClient.getUser(m.UserID)
	if e != nil {
		return e
	}

	t := m.Text + " (at <#" + m.ChannelID + ">)"
	// TODO about response
	_, e = service.SlackClient.postMessage(service.TimelineChannelID, t, u.Name, u.Profile.ImageURL)
	return e
}

func isPublic(channelID string) bool {
	return channelID[0:1] == "C"
}
<commit_msg>When message is first, it'll be ignored<commit_after>package timeline

import (
	"log"

	"github.com/syndtr/goleveldb/leveldb"
)

type TimelineService struct {
	SlackClient         slackClient
	TimelineChannelID   string
	BlackListChannelIDs []string
	db                  leveldb.DB
	logger              log.Logger
}

func NewTimelineService(slackAPIToken, timelineChannelID string, blackListChannelIDs []string, db leveldb.DB, logger log.Logger) TimelineService {
	return TimelineService{
		SlackClient:         slackClient{Token: slackAPIToken},
		TimelineChannelID:   timelineChannelID,
		BlackListChannelIDs: blackListChannelIDs,
		db:                  db,
		logger:              logger,
	}
}

func (service *TimelineService) Run() error {
	messageChan := make(chan *slackMessage)
	errorChan := make(chan error)

	go service.SlackClient.polling(messageChan, errorChan)
	isFirst := false
	for {
		select {
		case msg := <-messageChan:
			if msg.Type == "hello" {
				isFirst = true
			}
			if !service.isTargetMessage(msg) {
				service.logger.Println(msg)
				continue
			}
			key := msg.ChannelID + "-" + msg.TimeStamp
			_, err := service.db.Get([]byte(key), nil)
			if err == nil || isFirst {
				isFirst = false
				continue
			}
			e := service.postMessage(msg)
			if e != nil {
				service.db.Put([]byte(key), []byte(key), nil)
				return e
			}
		case e := <-errorChan:
			return e
		default:
			break
		}
	}
	return nil
}

func (service *TimelineService) isTargetMessage(m *slackMessage) bool {
	return m.Type == "message" &&
		m.ChannelID != service.TimelineChannelID &&
		isPublic(m.ChannelID) &&
		!contains(service.BlackListChannelIDs, m.ChannelID)
}

func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

func (service *TimelineService) postMessage(m *slackMessage) error {
	// TODO cache?
	u, e := service.SlackClient.getUser(m.UserID)
	if e != nil {
		return e
	}

	t := m.Text + " (at <#" + m.ChannelID + ">)"
	// TODO about response
	_, e = service.SlackClient.postMessage(service.TimelineChannelID, t, u.Name, u.Profile.ImageURL)
	return e
}

func isPublic(channelID string) bool {
	return channelID[0:1] == "C"
}
<|endoftext|>


<commit_before>// timeutil provides a set of time utilities including comparisons,
// conversion to "DT8" int32 and "DT14" int64 formats and other
// capabilities.
package timeutil

import (
	"fmt"
	"strings"
	"time"
)

const (
	DT14               = "20060102150405"
	DT6                = "200601"
	DT8                = "20060102"
	RFC3339Min         = "0000-01-01T00:00:00Z"
	RFC3339Max         = "9999-12-31T23:59:59Z"
	RFC3339Zero        = "0001-01-01T00:00:00Z"
	RFC3339YMD         = "2006-01-02"
	RFC3339YMDZeroUnix = int64(-62135596800)
	ISO8601YM          = "2006-01"
	ISO8601Z2          = "2006-01-02T15:04:05-07"
	ISO8601Z4          = "2006-01-02T15:04:05-0700"
	ISO8601ZCompact    = "20060102T150405Z"
	ISO8601NoTzMilli   = "2006-01-02T15:04:05.000"
	DateMDYSlash       = "01/02/2006"
	DMYHM2             = "02:01:06 15:04" // GMT time in format dd:mm:yy hh:mm
)

// Reformat a time string from one format to another
func FromTo(timeStringSrc, fromFormat, toFormat string) (string, error) {
	t, err := time.Parse(fromFormat, strings.TrimSpace(timeStringSrc))
	if err != nil {
		return "", err
	}
	return t.Format(toFormat), nil
}

// ParseFirst attempts to parse a string with a set of formats.
func ParseFirst(input string, formats ...string) (time.Time, error) {
	input = strings.TrimSpace(input)
	if len(formats) == 0 {
		return time.Now(), fmt.Errorf(
			"Requires value [%v] and at least one format [%v]", input, strings.Join(formats, ","))
	}
	for _, format := range formats {
		format = strings.TrimSpace(format)
		if len(format) == 0 {
			continue
		}
		if dt, err := time.Parse(input, format); err == nil {
			return dt, nil
		}
	}
	return time.Now(), fmt.Errorf("Cannot parse time [%v] with formats [%v]",
		input, strings.Join(formats, ","))
}

var FormatMap = map[string]string{
	"RFC3339":    time.RFC3339,
	"RFC3339YMD": RFC3339YMD,
	"ISO8601YM":  ISO8601YM,
}

func GetFormat(formatName string) (string, error) {
	format, ok := FormatMap[strings.TrimSpace(formatName)]
	if !ok {
		return "", fmt.Errorf("Format Not Found: %v", format)
	}
	return format, nil
}

// FormatQuarter takes quarter time and formats it using "Q# YYYY".
func FormatQuarter(t time.Time) string {
	return fmt.Sprintf("Q%d %d", MonthToQuarter(uint8(t.Month())), t.Year())
}

func TimeRFC3339Zero() time.Time {
	t0, _ := time.Parse(time.RFC3339, RFC3339Zero)
	return t0
}

type RFC3339YMDTime struct{ time.Time }

type ISO8601NoTzMilliTime struct{ time.Time }

func (t *RFC3339YMDTime) UnmarshalJSON(buf []byte) error {
	tt, isNil, err := timeUnmarshalJSON(buf, RFC3339YMD)
	if err != nil || isNil {
		return err
	}
	t.Time = tt
	return nil
}

func (t RFC3339YMDTime) MarshalJSON() ([]byte, error) {
	return timeMarshalJSON(t.Time, RFC3339YMD)
}

func (t *ISO8601NoTzMilliTime) UnmarshalJSON(buf []byte) error {
	tt, isNil, err := timeUnmarshalJSON(buf, ISO8601NoTzMilli)
	if err != nil || isNil {
		return err
	}
	t.Time = tt
	return nil
}

func (t ISO8601NoTzMilliTime) MarshalJSON() ([]byte, error) {
	return timeMarshalJSON(t.Time, ISO8601NoTzMilli)
}

func timeUnmarshalJSON(buf []byte, format string) (time.Time, bool, error) {
	str := string(buf)
	isNil := true
	if str == "null" || str == "\"\"" {
		return time.Time{}, isNil, nil
	}
	tt, err := time.Parse(format, strings.Trim(str, `"`))
	if err != nil {
		return time.Time{}, false, err
	}
	return tt, false, nil
}

func timeMarshalJSON(t time.Time, format string) ([]byte, error) {
	return []byte(`"` + t.Format(format) + `"`), nil
}
<commit_msg>update timeutil.ParseFirst signature<commit_after>// timeutil provides a set of time utilities including comparisons,
// conversion to "DT8" int32 and "DT14" int64 formats and other
// capabilities.
package timeutil

import (
	"fmt"
	"strings"
	"time"
)

const (
	DT14               = "20060102150405"
	DT6                = "200601"
	DT8                = "20060102"
	RFC3339Min         = "0000-01-01T00:00:00Z"
	RFC3339Max         = "9999-12-31T23:59:59Z"
	RFC3339Zero        = "0001-01-01T00:00:00Z"
	RFC3339YMD         = "2006-01-02"
	RFC3339YMDZeroUnix = int64(-62135596800)
	ISO8601YM          = "2006-01"
	ISO8601Z2          = "2006-01-02T15:04:05-07"
	ISO8601Z4          = "2006-01-02T15:04:05-0700"
	ISO8601ZCompact    = "20060102T150405Z"
	ISO8601NoTzMilli   = "2006-01-02T15:04:05.000"
	DateMDYSlash       = "01/02/2006"
	DMYHM2             = "02:01:06 15:04" // GMT time in format dd:mm:yy hh:mm
)

// Reformat a time string from one format to another
func FromTo(value, fromLayout, toLayout string) (string, error) {
	t, err := time.Parse(fromLayout, strings.TrimSpace(value))
	if err != nil {
		return "", err
	}
	return t.Format(toLayout), nil
}

// ParseFirst attempts to parse a string with a set of layouts.
func ParseFirst(layouts []string, value string) (time.Time, error) {
	value = strings.TrimSpace(value)
	if len(value) == 0 || len(layouts) == 0 {
		return time.Now(), fmt.Errorf(
			"Requires value [%v] and at least one layout [%v]", value, strings.Join(layouts, ","))
	}
	for _, layout := range layouts {
		layout = strings.TrimSpace(layout)
		if len(layout) == 0 {
			continue
		}
		if dt, err := time.Parse(layout, value); err == nil {
			return dt, nil
		}
	}
	return time.Now(), fmt.Errorf("Cannot parse time [%v] with layouts [%v]",
		value, strings.Join(layouts, ","))
}

var FormatMap = map[string]string{
	"RFC3339":    time.RFC3339,
	"RFC3339YMD": RFC3339YMD,
	"ISO8601YM":  ISO8601YM,
}

func GetFormat(formatName string) (string, error) {
	format, ok := FormatMap[strings.TrimSpace(formatName)]
	if !ok {
		return "", fmt.Errorf("Format Not Found: %v", format)
	}
	return format, nil
}

// FormatQuarter takes quarter time and formats it using "Q# YYYY".
func FormatQuarter(t time.Time) string {
	return fmt.Sprintf("Q%d %d", MonthToQuarter(uint8(t.Month())), t.Year())
}

func TimeRFC3339Zero() time.Time {
	t0, _ := time.Parse(time.RFC3339, RFC3339Zero)
	return t0
}

type RFC3339YMDTime struct{ time.Time }

type ISO8601NoTzMilliTime struct{ time.Time }

func (t *RFC3339YMDTime) UnmarshalJSON(buf []byte) error {
	tt, isNil, err := timeUnmarshalJSON(buf, RFC3339YMD)
	if err != nil || isNil {
		return err
	}
	t.Time = tt
	return nil
}

func (t RFC3339YMDTime) MarshalJSON() ([]byte, error) {
	return timeMarshalJSON(t.Time, RFC3339YMD)
}

func (t *ISO8601NoTzMilliTime) UnmarshalJSON(buf []byte) error {
	tt, isNil, err := timeUnmarshalJSON(buf, ISO8601NoTzMilli)
	if err != nil || isNil {
		return err
	}
	t.Time = tt
	return nil
}

func (t ISO8601NoTzMilliTime) MarshalJSON() ([]byte, error) {
	return timeMarshalJSON(t.Time, ISO8601NoTzMilli)
}

func timeUnmarshalJSON(buf []byte, layout string) (time.Time, bool, error) {
	str := string(buf)
	isNil := true
	if str == "null" || str == "\"\"" {
		return time.Time{}, isNil, nil
	}
	tt, err := time.Parse(layout, strings.Trim(str, `"`))
	if err != nil {
		return time.Time{}, false, err
	}
	return tt, false, nil
}

func timeMarshalJSON(t time.Time, layout string) ([]byte, error) {
	return []byte(`"` + t.Format(layout) + `"`), nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Compose, an IBM Company
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package composeapi

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/parnurzeal/gorequest"
)

// Deployment structure
type Deployment struct {
	ID                  string            `json:"id"`
	Name                string            `json:"name"`
	Type                string            `json:"type"`
	CreatedAt           time.Time         `json:"created_at"`
	ProvisionRecipeID   string            `json:"provision_recipe_id,omitempty"`
	CACertificateBase64 string            `json:"ca_certificate_base64,omitempty"`
	Connection          ConnectionStrings `json:"connection_strings,omitempty"`
	Links               struct {
		ComposeWebUILink Link `json:"compose_web_ui"`
	} `json:"_links"`
}

// ConnectionStrings structure, part of the Deployment struct
type ConnectionStrings struct {
	Health   string   `json:"health,omitempty"`
	SSH      string   `json:"ssh,omitempty"`
	Admin    string   `json:"admin,omitempty"`
	SSHAdmin string   `json:"ssh_admin,omitempty"`
	CLI      []string `json:"cli,omitempty"`
	Direct   []string `json:"direct,omitempty"`
}

// deploymentsResource is used to represent and remove the JSON+HAL Embedded wrapper
type deploymentsResponse struct {
	Embedded struct {
		Deployments []Deployment `json:"deployments"`
	} `json:"_embedded"`
}

//CreateDeploymentParams Parameters to be completed before creating a deployment
type createDeploymentParams struct {
	Deployment DeploymentParams `json:"deployment"`
}

// DeploymentParams core parameters for a new deployment
type DeploymentParams struct {
	Name         string `json:"name"`
	AccountID    string `json:"account_id"`
	ClusterID    string `json:"cluster_id,omitempty"`
	Datacenter   string `json:"datacenter,omitempty"`
	DatabaseType string `json:"type"`
	Version      string `json:"version,omitempty"`
	Units        int    `json:"units,omitempty"`
	SSL          bool   `json:"ssl,omitempty"`
	WiredTiger   bool   `json:"wired_tiger,omitempty"`
}

//VersionTransition a struct wrapper for version transition information
type VersionTransition struct {
	Application string `json:"application"`
	Method      string `json:"method"`
	FromVersion string `json:"from_version"`
	ToVersion   string `json:"to_version"`
}

type versionsResponse struct {
	Embedded struct {
		VersionTransitions []VersionTransition `json:"transitions"`
	} `json:"_embedded"`
}

//CreateDeploymentJSON performs the call
func (c *Client) CreateDeploymentJSON(params DeploymentParams) (string, []error) {
	deploymentparams := createDeploymentParams{Deployment: params}

	response, body, errs := gorequest.New().Post(apibase+"deployments").
		Set("Authorization", "Bearer "+c.apiToken).
		Set("Content-type", "application/json; charset=utf-8").
		Send(deploymentparams).
		End()

	if response.StatusCode != 202 { // Expect Accepted on success - assume error on anything else
		myerrors := Errors{}
		err := json.Unmarshal([]byte(body), &myerrors)
		if err != nil {
			errs = append(errs, fmt.Errorf("Unable to parse error - status code %d - body %s", response.StatusCode, response.Body))
		} else {
			errs = append(errs, fmt.Errorf("%v", myerrors.Error))
		}
	}

	return body, errs
}

//CreateDeployment creates a deployment
func (c *Client) CreateDeployment(params DeploymentParams) (*Deployment, []error) {

	// This is a POST not a GET, so it builds its own request

	body, errs := c.CreateDeploymentJSON(params)

	if errs != nil {
		return nil, errs
	}

	deployed := Deployment{}
	json.Unmarshal([]byte(body), &deployed)

	return &deployed, nil
}

//GetDeploymentsJSON returns raw deployment
func (c *Client) GetDeploymentsJSON() (string, []error) { return c.getJSON("deployments") }

//GetDeployments returns deployment structure
func (c *Client) GetDeployments() (*[]Deployment, []error) {
	body, errs := c.GetDeploymentsJSON()

	if errs != nil {
		return nil, errs
	}

	deploymentResponse := deploymentsResponse{}
	json.Unmarshal([]byte(body), &deploymentResponse)
	deployments := deploymentResponse.Embedded.Deployments

	return &deployments, nil
}

//GetDeploymentJSON returns raw deployment
func (c *Client) GetDeploymentJSON(deploymentid string) (string, []error) {
	return c.getJSON("deployments/" + deploymentid)
}

//GetDeployment returns deployment structure
func (c *Client) GetDeployment(deploymentid string) (*Deployment, []error) {
	body, errs := c.GetDeploymentJSON(deploymentid)

	if errs != nil {
		return nil, errs
	}

	deployment := Deployment{}
	json.Unmarshal([]byte(body), &deployment)

	return &deployment, nil
}
<commit_msg>Added missing Deprovision Deployment (#5)<commit_after>// Copyright 2016 Compose, an IBM Company
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package composeapi

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/parnurzeal/gorequest"
)

// Deployment structure
type Deployment struct {
	ID                  string            `json:"id"`
	Name                string            `json:"name"`
	Type                string            `json:"type"`
	CreatedAt           time.Time         `json:"created_at"`
	ProvisionRecipeID   string            `json:"provision_recipe_id,omitempty"`
	CACertificateBase64 string            `json:"ca_certificate_base64,omitempty"`
	Connection          ConnectionStrings `json:"connection_strings,omitempty"`
	Links               struct {
		ComposeWebUILink Link `json:"compose_web_ui"`
	} `json:"_links"`
}

// ConnectionStrings structure, part of the Deployment struct
type ConnectionStrings struct {
	Health   string   `json:"health,omitempty"`
	SSH      string   `json:"ssh,omitempty"`
	Admin    string   `json:"admin,omitempty"`
	SSHAdmin string   `json:"ssh_admin,omitempty"`
	CLI      []string `json:"cli,omitempty"`
	Direct   []string `json:"direct,omitempty"`
}

// deploymentsResource is used to represent and remove the JSON+HAL Embedded wrapper
type deploymentsResponse struct {
	Embedded struct {
		Deployments []Deployment `json:"deployments"`
	} `json:"_embedded"`
}

//CreateDeploymentParams Parameters to be completed before creating a deployment
type CreateDeploymentParams struct {
	Deployment DeploymentParams `json:"deployment"`
}

// DeploymentParams core parameters for a new deployment
type DeploymentParams struct {
	Name         string `json:"name"`
	AccountID    string `json:"account_id"`
	ClusterID    string `json:"cluster_id,omitempty"`
	Datacenter   string `json:"datacenter,omitempty"`
	DatabaseType string `json:"type"`
	Version      string `json:"version,omitempty"`
	Units        int    `json:"units,omitempty"`
	SSL          bool   `json:"ssl,omitempty"`
	WiredTiger   bool   `json:"wired_tiger,omitempty"`
}

//VersionTransition a struct wrapper for version transition information
type VersionTransition struct {
	Application string `json:"application"`
	Method      string `json:"method"`
	FromVersion string `json:"from_version"`
	ToVersion   string `json:"to_version"`
}

type versionsResponse struct {
	Embedded struct {
		VersionTransitions []VersionTransition `json:"transitions"`
	} `json:"_embedded"`
}

//CreateDeploymentJSON performs the call
func (c *Client) CreateDeploymentJSON(params DeploymentParams) (string, []error) {
	deploymentparams := CreateDeploymentParams{Deployment: params}

	response, body, errs := gorequest.New().Post(apibase+"deployments").
		Set("Authorization", "Bearer "+c.apiToken).
		Set("Content-type", "application/json; charset=utf-8").
		Send(deploymentparams).
		End()

	if response.StatusCode != 202 { // Expect Accepted on success - assume error on anything else
		myerrors := Errors{}
		err := json.Unmarshal([]byte(body), &myerrors)
		if err != nil {
			errs = append(errs, fmt.Errorf("Unable to parse error - status code %d - body %s", response.StatusCode, response.Body))
		} else {
			errs = append(errs, fmt.Errorf("%v", myerrors.Error))
		}
	}

	return body, errs
}

//CreateDeployment creates a deployment
func (c *Client) CreateDeployment(params DeploymentParams) (*Deployment, []error) {

	// This is a POST not a GET, so it builds its own request

	body, errs := c.CreateDeploymentJSON(params)

	if errs != nil {
		return nil, errs
	}

	deployed := Deployment{}
	json.Unmarshal([]byte(body), &deployed)

	return &deployed, nil
}

//GetDeploymentsJSON returns raw deployment
func (c *Client) GetDeploymentsJSON() (string, []error) { return c.getJSON("deployments") }

//GetDeployments returns deployment structure
func (c *Client) GetDeployments() (*[]Deployment, []error) {
	body, errs := c.GetDeploymentsJSON()

	if errs != nil {
		return nil, errs
	}

	deploymentResponse := deploymentsResponse{}
	json.Unmarshal([]byte(body), &deploymentResponse)
	deployments := deploymentResponse.Embedded.Deployments

	return &deployments, nil
}

//GetDeploymentJSON returns raw deployment
func (c *Client) GetDeploymentJSON(deploymentid string) (string, []error) {
	return c.getJSON("deployments/" + deploymentid)
}

//GetDeployment returns deployment structure
func (c *Client) GetDeployment(deploymentid string) (*Deployment, []error) {
	body, errs := c.GetDeploymentJSON(deploymentid)

	if errs != nil {
		return nil, errs
	}

	deployment := Deployment{}
	json.Unmarshal([]byte(body), &deployment)

	return &deployment, nil
}

//DeprovisionDeploymentJSON performs the call
func (c *Client) DeprovisionDeploymentJSON(deploymentID string) (string, []error) {

	response, body, errs := gorequest.New().Delete(apibase+"deployments/"+deploymentID).
		Set("Authorization", "Bearer "+c.apiToken).
		Set("Content-type", "application/json; charset=utf-8").
		End()

	if response.StatusCode != 202 { // Expect Accepted on success - assume error on anything else
		myerrors := Errors{}
		err := json.Unmarshal([]byte(body), &myerrors)
		if err != nil {
			errs = append(errs, fmt.Errorf("Unable to parse error - status code %d - body %s", response.StatusCode, response.Body))
		} else {
			errs = append(errs, fmt.Errorf("%v", myerrors.Error))
		}
	}

	return body, errs
}

//DeprovisionDeployment deletes a deployment
func (c *Client) DeprovisionDeployment(deploymentID string) (*Recipe, []error) {

	// This is a POST not a GET, so it builds its own request

	body, errs := c.DeprovisionDeploymentJSON(deploymentID)

	if errs != nil {
		return nil, errs
	}

	deprovrecipe := Recipe{}
	json.Unmarshal([]byte(body), &deprovrecipe)

	return &deprovrecipe, nil
}
<|endoftext|>


<commit_before>package isogrids

import (
	"image/color"
	"math"
	"net/http"

	"github.com/ajstarks/svgo"
	"github.com/taironas/tinygraphs/colors"
	"github.com/taironas/tinygraphs/draw"
)

// RandomGradientSVG builds an image.
func RandomGradientSVG(w http.ResponseWriter, colors, gColors []color.RGBA, gv colors.GradientVector, width, height, xsquares int) {

	var gradientColors []svg.Offcolor
	gradientColors = make([]svg.Offcolor, len(gColors))
	percentage := uint8(100 / len(gColors))

	step := uint8(100 / len(gColors))
	for i, c := range gColors {
		gradientColors[i] = svg.Offcolor{percentage, RGBToHex(c.R, c.G, c.B), 1}
		percentage += step
	}

	canvas := svg.New(w)
	canvas.Start(width, height)
	canvas.Def()
	canvas.LinearGradient("gradientColors", gv.X1, gv.Y1, gv.X2, gv.Y2, gradientColors)
	canvas.DefEnd()
	canvas.Rect(0, 0, width, height, "fill:url(#gradientColors)")

	squares := xsquares
	quadrantSize := width / squares
	middle := math.Ceil(float64(squares) / float64(2))
	colorMap := make(map[int]color.RGBA)
	colorIndex := make(map[int]int)
	for yQ := 0; yQ < squares; yQ++ {
		y := yQ * quadrantSize
		colorMap = make(map[int]color.RGBA)
		colorIndex = make(map[int]int)
		for xQ := 0; xQ < squares; xQ++ {
			x := xQ * quadrantSize
			fill := ""
			if _, ok := colorMap[xQ]; !ok {
				if float64(xQ) < middle {
					colorIndex[xQ] = draw.RandomIndexFromArray(colors)
					colorMap[xQ] = colors[colorIndex[xQ]]
				} else if xQ < squares {
					colorIndex[xQ] = colorIndex[squares-xQ-1]
					colorMap[xQ] = colorMap[squares-xQ-1]
				} else {
					colorIndex[xQ] = colorIndex[0]
					colorMap[xQ] = colorMap[0]
				}
			}
			if colorIndex[xQ] != 0 {
				fill = "fill:none"
			} else {
				fill = draw.FillFromRGBA(colorMap[xQ])

			}
			canvas.Rect(x, y, quadrantSize, quadrantSize, fill)
		}
	}
	canvas.End()
}
<commit_msg>fix typo #67<commit_after>package isogrids

import (
	"image/color"
	"math"
	"net/http"

	"github.com/ajstarks/svgo"
	"github.com/taironas/tinygraphs/colors"
	"github.com/taironas/tinygraphs/draw"
)

// RandomGradientSVG builds an image.
func RandomGradientSVG(w http.ResponseWriter, colors, gColors []color.RGBA, gv colors.GradientVector, width, height, xsquares int) {

	var gradientColors []svg.Offcolor
	gradientColors = make([]svg.Offcolor, len(gColors))
	percentage := uint8(100 / len(gColors))

	step := uint8(100 / len(gColors))
	for i, c := range gColors {
		gradientColors[i] = svg.Offcolor{percentage, draw.RGBToHex(c.R, c.G, c.B), 1}
		percentage += step
	}

	canvas := svg.New(w)
	canvas.Start(width, height)
	canvas.Def()
	canvas.LinearGradient("gradientColors", gv.X1, gv.Y1, gv.X2, gv.Y2, gradientColors)
	canvas.DefEnd()
	canvas.Rect(0, 0, width, height, "fill:url(#gradientColors)")

	squares := xsquares
	quadrantSize := width / squares
	middle := math.Ceil(float64(squares) / float64(2))
	colorMap := make(map[int]color.RGBA)
	colorIndex := make(map[int]int)
	for yQ := 0; yQ < squares; yQ++ {
		y := yQ * quadrantSize
		colorMap = make(map[int]color.RGBA)
		colorIndex = make(map[int]int)
		for xQ := 0; xQ < squares; xQ++ {
			x := xQ * quadrantSize
			fill := ""
			if _, ok := colorMap[xQ]; !ok {
				if float64(xQ) < middle {
					colorIndex[xQ] = draw.RandomIndexFromArray(colors)
					colorMap[xQ] = colors[colorIndex[xQ]]
				} else if xQ < squares {
					colorIndex[xQ] = colorIndex[squares-xQ-1]
					colorMap[xQ] = colorMap[squares-xQ-1]
				} else {
					colorIndex[xQ] = colorIndex[0]
					colorMap[xQ] = colorMap[0]
				}
			}
			if colorIndex[xQ] != 0 {
				fill = "fill:none"
			} else {
				fill = draw.FillFromRGBA(colorMap[xQ])

			}
			canvas.Rect(x, y, quadrantSize, quadrantSize, fill)
		}
	}
	canvas.End()
}
<|endoftext|>


<commit_before>package tokenizer

import (
	"bufio"
	"io"

	gzip "github.com/klauspost/pgzip"
)

type GzLineReader struct {
	in io.Reader
	gr *gzip.Reader
	br *bufio.Reader
}

func NewGzLine(in io.Reader) (*GzLineReader, error) {
	l := &GzLineReader{in: in}
	var err error
	l.gr, err = gzip.NewReader(in)
	if err != nil {
		return nil, err
	}
	l.br = bufio.NewReader(l.gr)
	return l, nil
}

func (l *GzLineReader) Next() (string, error) {
	record, err := l.br.ReadBytes(10)
	if err != nil {
		return "", err
	}
	return string(record), nil
}

func (l *GzLineReader) Close() error {
	return l.gr.Close()
}
<commit_msg>Add uncompressed line reader.<commit_after>// Tokenizers for various formats,
// that satisfies the password.Tokenizer interface.
package tokenizer

import (
	"bufio"
	"io"

	gzip "github.com/klauspost/pgzip"
)

type LineReader struct {
	in io.Reader
	gr *gzip.Reader
	br *bufio.Reader
}

// NewLine reads one password per line until
// \0xa (newline) is encountered.
// Input is streamed.
func NewLine(in io.Reader) (*LineReader, error) {
	l := &LineReader{in: in}
	l.br = bufio.NewReader(in)
	return l, nil
}

// NewGzLine reads one password per line until
// \0xa (newline) is encountered.
// The input is assumed to be gzip compressed.
// Input is streamed.
func NewGzLine(in io.Reader) (*LineReader, error) {
	l := &LineReader{in: in}
	var err error
	l.gr, err = gzip.NewReader(in)
	if err != nil {
		return nil, err
	}
	l.br = bufio.NewReader(l.gr)
	return l, nil
}

func (l *LineReader) Next() (string, error) {
	record, err := l.br.ReadBytes(10)
	if err != nil {
		return "", err
	}
	return string(record), nil
}

func (l *LineReader) Close() error {
	if l.gr != nil {
		return l.gr.Close()
	}
	return nil
}
<|endoftext|>


<commit_before>package graft

import (
	"encoding/json"
	"io/ioutil"
	"os"
)

type checksum struct {
	sha, data []byte
}

type persistentState struct {
	CurrentTerm uint64
	VotedFor    string
}

func (n *Node) closeLog() error {
	err := os.Remove(n.logPath)
	n.logPath = ""
	return err
}

func (n *Node) initLog(path string) error {
	if _, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0660); err != nil {
		return err
	}
	n.logPath = path
	ps, err := n.readState(path)
	if err != nil || err != ErrLogNoState {
		return err
	}
	if ps != nil {
		n.setTerm(ps.CurrentTerm)
		n.setVote(ps.VotedFor)
	}
	return nil

}

func (n *Node) readState(path string) (*persistentState, error) {
	buf, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	if len(buf) <= 0 {
		return nil, ErrLogNoState
	}

	ps := &persistentState{}
	if err := json.Unmarshal(buf, ps); err != nil {
		return nil, err
	}
	return ps, nil
}

func (n *Node) writeState() error {
	n.mu.Lock()
	ps := persistentState{
		CurrentTerm: n.term,
		VotedFor:    n.vote,
	}
	logPath := n.logPath
	n.mu.Unlock()
	buf, err := json.Marshal(ps)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(logPath, buf, 0660)

}
<commit_msg>fix bug<commit_after>package graft

import (
	"encoding/json"
	"io/ioutil"
	"os"
)

type checksum struct {
	sha, data []byte
}

type persistentState struct {
	CurrentTerm uint64
	VotedFor    string
}

func (n *Node) closeLog() error {
	err := os.Remove(n.logPath)
	n.logPath = ""
	return err
}

func (n *Node) initLog(path string) error {
	file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0660)
	if err != nil {
		return err
	}
	defer file.Close()
	n.logPath = path
	ps, err := n.readState(path)
	if err != nil && err != ErrLogNoState {
		return err
	}
	if ps != nil {
		n.setTerm(ps.CurrentTerm)
		n.setVote(ps.VotedFor)
	}
	return nil

}

func (n *Node) readState(path string) (*persistentState, error) {
	buf, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	if len(buf) <= 0 {
		return nil, ErrLogNoState
	}

	ps := &persistentState{}
	if err := json.Unmarshal(buf, ps); err != nil {
		return nil, err
	}
	return ps, nil
}

func (n *Node) writeState() error {
	n.mu.Lock()
	ps := persistentState{
		CurrentTerm: n.term,
		VotedFor:    n.vote,
	}
	logPath := n.logPath
	n.mu.Unlock()
	buf, err := json.Marshal(ps)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(logPath, buf, 0660)

}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013, 2014 Conformal Systems LLC <info@conformal.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package main

import (
	"fmt"
	"os"

	"github.com/conformal/btclog"
	"github.com/conformal/btcwallet/chain"
	"github.com/conformal/btcwallet/txstore"
	"github.com/conformal/seelog"
)

const (
	// lockTimeThreshold is the number below which a lock time is
	// interpreted to be a block number.  Since an average of one block
	// is generated per 10 minutes, this allows blocks for about 9,512
	// years.  However, if the field is interpreted as a timestamp, given
	// the lock time is a uint32, the max is sometime around 2106.
	lockTimeThreshold uint32 = 5e8 // Tue Nov 5 00:53:20 1985 UTC
)

// Loggers per subsytem.  Note that backendLog is a seelog logger that all of
// the subsystem loggers route their messages to.  When adding new subsystems,
// add a reference here, to the subsystemLoggers map, and the useLogger
// function.
var (
	backendLog = seelog.Disabled
	log        = btclog.Disabled
	txstLog    = btclog.Disabled
	chainLog   = btclog.Disabled
)

// subsystemLoggers maps each subsystem identifier to its associated logger.
var subsystemLoggers = map[string]btclog.Logger{
	"BTCW": log,
	"TXST": txstLog,
	"CHNS": chainLog,
}

// logClosure is used to provide a closure over expensive logging operations
// so don't have to be performed when the logging level doesn't warrant it.
type logClosure func() string

// String invokes the underlying function and returns the result.
func (c logClosure) String() string {
	return c()
}

// newLogClosure returns a new closure over a function that returns a string
// which itself provides a Stringer interface so that it can be used with the
// logging system.
func newLogClosure(c func() string) logClosure {
	return logClosure(c)
}

// useLogger updates the logger references for subsystemID to logger.  Invalid
// subsystems are ignored.
func useLogger(subsystemID string, logger btclog.Logger) {
	if _, ok := subsystemLoggers[subsystemID]; !ok {
		return
	}
	subsystemLoggers[subsystemID] = logger

	switch subsystemID {
	case "BTCW":
		log = logger
	case "TXST":
		txstLog = logger
		txstore.UseLogger(logger)
	case "CHNS":
		chainLog = logger
		chain.UseLogger(logger)
	}
}

// initSeelogLogger initializes a new seelog logger that is used as the backend
// for all logging subsytems.
func initSeelogLogger(logFile string) {
	config := `
        <seelog type="adaptive" mininterval="2000000" maxinterval="100000000"
                critmsgcount="500" minlevel="trace">
                <outputs formatid="all">
                        <console />
                        <rollingfile type="size" filename="%s" maxsize="10485760" maxrolls="3" />
                </outputs>
                <formats>
                        <format id="all" format="%%Time %%Date [%%LEV] %%Msg%%n" />
                </formats>
        </seelog>`
	config = fmt.Sprintf(config, logFile)

	logger, err := seelog.LoggerFromConfigAsString(config)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to create logger: %v", err)
		os.Exit(1)
	}

	backendLog = logger
}

// setLogLevel sets the logging level for provided subsystem.  Invalid
// subsystems are ignored.  Uninitialized subsystems are dynamically created as
// needed.
func setLogLevel(subsystemID string, logLevel string) {
	// Ignore invalid subsystems.
	logger, ok := subsystemLoggers[subsystemID]
	if !ok {
		return
	}

	// Default to info if the log level is invalid.
	level, ok := btclog.LogLevelFromString(logLevel)
	if !ok {
		level = btclog.InfoLvl
	}

	// Create new logger for the subsystem if needed.
	if logger == btclog.Disabled {
		logger = btclog.NewSubsystemLogger(backendLog, subsystemID+": ")
		useLogger(subsystemID, logger)
	}
	logger.SetLevel(level)
}

// setLogLevels sets the log level for all subsystem loggers to the passed
// level.  It also dynamically creates the subsystem loggers as needed, so it
// can be used to initialize the logging system.
func setLogLevels(logLevel string) {
	// Configure all sub-systems with the new logging level.  Dynamically
	// create loggers as needed.
	for subsystemID := range subsystemLoggers {
		setLogLevel(subsystemID, logLevel)
	}
}

// pickNoun returns the singular or plural form of a noun depending
// on the count n.
func pickNoun(n int, singular, plural string) string {
	if n == 1 {
		return singular
	}
	return plural
}
<commit_msg>Update seelog import paths to new location.<commit_after>/*
 * Copyright (c) 2013, 2014 Conformal Systems LLC <info@conformal.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package main

import (
	"fmt"
	"os"

	"github.com/btcsuite/seelog"
	"github.com/conformal/btclog"
	"github.com/conformal/btcwallet/chain"
	"github.com/conformal/btcwallet/txstore"
)

const (
	// lockTimeThreshold is the number below which a lock time is
	// interpreted to be a block number.  Since an average of one block
	// is generated per 10 minutes, this allows blocks for about 9,512
	// years.  However, if the field is interpreted as a timestamp, given
	// the lock time is a uint32, the max is sometime around 2106.
	lockTimeThreshold uint32 = 5e8 // Tue Nov 5 00:53:20 1985 UTC
)

// Loggers per subsytem.  Note that backendLog is a seelog logger that all of
// the subsystem loggers route their messages to.  When adding new subsystems,
// add a reference here, to the subsystemLoggers map, and the useLogger
// function.
var (
	backendLog = seelog.Disabled
	log        = btclog.Disabled
	txstLog    = btclog.Disabled
	chainLog   = btclog.Disabled
)

// subsystemLoggers maps each subsystem identifier to its associated logger.
var subsystemLoggers = map[string]btclog.Logger{
	"BTCW": log,
	"TXST": txstLog,
	"CHNS": chainLog,
}

// logClosure is used to provide a closure over expensive logging operations
// so don't have to be performed when the logging level doesn't warrant it.
type logClosure func() string

// String invokes the underlying function and returns the result.
func (c logClosure) String() string {
	return c()
}

// newLogClosure returns a new closure over a function that returns a string
// which itself provides a Stringer interface so that it can be used with the
// logging system.
func newLogClosure(c func() string) logClosure {
	return logClosure(c)
}

// useLogger updates the logger references for subsystemID to logger.  Invalid
// subsystems are ignored.
func useLogger(subsystemID string, logger btclog.Logger) {
	if _, ok := subsystemLoggers[subsystemID]; !ok {
		return
	}
	subsystemLoggers[subsystemID] = logger

	switch subsystemID {
	case "BTCW":
		log = logger
	case "TXST":
		txstLog = logger
		txstore.UseLogger(logger)
	case "CHNS":
		chainLog = logger
		chain.UseLogger(logger)
	}
}

// initSeelogLogger initializes a new seelog logger that is used as the backend
// for all logging subsytems.
func initSeelogLogger(logFile string) {
	config := `
        <seelog type="adaptive" mininterval="2000000" maxinterval="100000000"
                critmsgcount="500" minlevel="trace">
                <outputs formatid="all">
                        <console />
                        <rollingfile type="size" filename="%s" maxsize="10485760" maxrolls="3" />
                </outputs>
                <formats>
                        <format id="all" format="%%Time %%Date [%%LEV] %%Msg%%n" />
                </formats>
        </seelog>`
	config = fmt.Sprintf(config, logFile)

	logger, err := seelog.LoggerFromConfigAsString(config)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to create logger: %v", err)
		os.Exit(1)
	}

	backendLog = logger
}

// setLogLevel sets the logging level for provided subsystem.  Invalid
// subsystems are ignored.  Uninitialized subsystems are dynamically created as
// needed.
func setLogLevel(subsystemID string, logLevel string) {
	// Ignore invalid subsystems.
	logger, ok := subsystemLoggers[subsystemID]
	if !ok {
		return
	}

	// Default to info if the log level is invalid.
	level, ok := btclog.LogLevelFromString(logLevel)
	if !ok {
		level = btclog.InfoLvl
	}

	// Create new logger for the subsystem if needed.
	if logger == btclog.Disabled {
		logger = btclog.NewSubsystemLogger(backendLog, subsystemID+": ")
		useLogger(subsystemID, logger)
	}
	logger.SetLevel(level)
}

// setLogLevels sets the log level for all subsystem loggers to the passed
// level.  It also dynamically creates the subsystem loggers as needed, so it
// can be used to initialize the logging system.
func setLogLevels(logLevel string) {
	// Configure all sub-systems with the new logging level.  Dynamically
	// create loggers as needed.
	for subsystemID := range subsystemLoggers {
		setLogLevel(subsystemID, logLevel)
	}
}

// pickNoun returns the singular or plural form of a noun depending
// on the count n.
func pickNoun(n int, singular, plural string) string {
	if n == 1 {
		return singular
	}
	return plural
}
<|endoftext|>


<commit_before>package emperror

// LogHandler accepts an errorLogger instance and logs an error.
//
// Compatible with most level-based loggers.
type LogHandler struct {
	logger errorLogger
}

// errorLogger covers most of the level-based logging solutions.
type errorLogger interface {
	Error(args ...interface{})
}

// NewLogHandler returns a new LogHandler.
func NewLogHandler(logger errorLogger) Handler {
	return &LogHandler{logger}
}

// Handle takes care of an error by logging it.
func (h *LogHandler) Handle(err error) {
	h.logger.Error(err)
}
<commit_msg>Rename errorLogger interface<commit_after>package emperror

// LogHandler accepts an logger instance and logs an error.
//
// Compatible with most level-based loggers.
type LogHandler struct {
	l logger
}

// logger covers most of the level-based logging solutions.
type logger interface {
	Error(args ...interface{})
}

// NewLogHandler returns a new LogHandler.
func NewLogHandler(l logger) Handler {
	return &LogHandler{l}
}

// Handle takes care of an error by logging it.
func (h *LogHandler) Handle(err error) {
	h.l.Error(err)
}
<|endoftext|>


<commit_before>package logged

import (
	"runtime"
	"strings"
	"time"
)

const (
	Info  = "info"
	Debug = "debug"
)

type Log interface {
	Info(message string, data map[string]string) error
	Debug(message string, data map[string]string) error
	IsDebug() bool
}

type Config struct {
	Serializer    Serializer
	DebugPackages []string
	Defaults      map[string]string
}

func New(c *Config) Log {
	return &log{
		serializer:    c.Serializer,
		debugPackages: c.DebugPackages,
		defaults:      c.Defaults,
	}
}

type log struct {
	serializer    Serializer
	defaults      map[string]string
	debugPackages []string
}

func (l *log) Info(message string, data map[string]string) error {
	return l.write(Info, message, data)
}

func (l *log) Debug(message string, data map[string]string) error {
	if l.IsDebug() {
		return l.write(Debug, message, data)
	}
	return nil
}

func (l *log) IsDebug() bool {
	if len(l.debugPackages) == 0 {
		return false
	}

	pc, _, _, _ := runtime.Caller(2)
	funcName := runtime.FuncForPC(pc).Name()

	for _, pkg := range l.debugPackages {
		if strings.HasPrefix(funcName, pkg) {
			return true
		}
	}

	return false
}

func (l *log) write(level, message string, data map[string]string) error {
	return l.serializer.Write(&Entry{
		Timestamp: time.Now().UTC().Format(time.RFC3339Nano),
		Level:     level,
		Message:   message,
		Data:      l.mergedData(data),
	})
}

func (l *log) mergedData(data map[string]string) map[string]string {
	if l.defaults == nil || len(l.defaults) == 0 {
		return data
	}

	if data == nil || len(data) == 0 {
		return l.defaults
	}

	merged := make(map[string]string)
	for k, v := range l.defaults {
		merged[k] = v
	}
	for k, v := range data {
		merged[k] = v
	}

	return merged
}
<commit_msg>Add wildcard debug package<commit_after>package logged

import (
	"runtime"
	"strings"
	"time"
)

const (
	Info  = "info"
	Debug = "debug"
)

type Log interface {
	Info(message string, data map[string]string) error
	Debug(message string, data map[string]string) error
	IsDebug() bool
}

type Config struct {
	Serializer    Serializer
	DebugPackages []string
	Defaults      map[string]string
}

func New(c *Config) Log {
	return &log{
		serializer:    c.Serializer,
		debugPackages: c.DebugPackages,
		defaults:      c.Defaults,
	}
}

type log struct {
	serializer    Serializer
	defaults      map[string]string
	debugPackages []string
}

func (l *log) Info(message string, data map[string]string) error {
	return l.write(Info, message, data)
}

func (l *log) Debug(message string, data map[string]string) error {
	if l.IsDebug() {
		return l.write(Debug, message, data)
	}
	return nil
}

func (l *log) IsDebug() bool {
	if len(l.debugPackages) == 0 {
		return false
	}

	if l.debugPackages[0] == "*" {
		return true
	}

	pc, _, _, _ := runtime.Caller(2)
	funcName := runtime.FuncForPC(pc).Name()

	for _, pkg := range l.debugPackages {
		if strings.HasPrefix(funcName, pkg) {
			return true
		}
	}

	return false
}

func (l *log) write(level, message string, data map[string]string) error {
	return l.serializer.Write(&Entry{
		Timestamp: time.Now().UTC().Format(time.RFC3339Nano),
		Level:     level,
		Message:   message,
		Data:      l.mergedData(data),
	})
}

func (l *log) mergedData(data map[string]string) map[string]string {
	if l.defaults == nil || len(l.defaults) == 0 {
		return data
	}

	if data == nil || len(data) == 0 {
		return l.defaults
	}

	merged := make(map[string]string)
	for k, v := range l.defaults {
		merged[k] = v
	}
	for k, v := range data {
		merged[k] = v
	}

	return merged
}
<|endoftext|>


<commit_before>package asyncLog

import (
	"encoding/json"
	"log"
	"math/rand"
	"os"
	"strings"
	"sync"
	"time"
)

type asyncLogType struct {
	files map[string]*LogFile // 下标是文件名

	// 避免并发new对象
	sync.Mutex
}

type LogFile struct {
	filename   string     // 原始文件名（包含完整目录）
	flag       int        // 默认为log.LstdFlags
	writeMutex sync.Mutex // 写文件时的锁

	// 同步设置
	sync struct {
		duration  time.Duration // 同步数据到文件的周期，默认为1秒
		beginTime time.Time     // 开始同步的时间，判断同步的耗时
		status    syncStatus    // 同步状态
	}

	// 日志的等级
	level Priority

	// 缓存
	cache struct {
		use  bool     // 是否使用缓存
		data []string // 缓存数据

		// 写cache时的互斥锁
		writeMutex sync.Mutex
	}

	// 文件切割
	logRotate struct {
		rotate LogRotate  // 默认按小时切割
		file   *os.File   // 文件操作对象
		suffix string     // 切割后的文件名后缀
		mutex  sync.Mutex // 文件名锁
	}

	// 日志写入概率
	probability float32
}

// log同步的状态
type syncStatus int

const (
	statusInit  syncStatus = iota // 初始状态
	statusDoing                   // 同步中
	statusDone                    // 同步已经完成
)

// 日志切割的方式
type LogRotate int

const (
	RotateHour LogRotate = iota // 按小时切割
	RotateDate                  // 按日期切割
)

const (
	// 写日志时前缀的时间格式
	// "2006-01-02T15:04:05Z07:00"
	logTimeFormat string = time.RFC3339

	// 文件写入mode
	fileOpenMode = 0666

	// 文件Flag
	fileFlag = os.O_WRONLY | os.O_CREATE | os.O_APPEND

	// 换行符
	newlineChar = "\n"

	// 缓存切片的初始容量
	cacheInitCap = 64
)

// 是否需要Flag信息
const (
	NoFlag  = 0
	StdFlag = log.LstdFlags
)

// 异步日志变量
var asyncLog *asyncLogType

var nowFunc = time.Now

func init() {
	asyncLog = &asyncLogType{
		files: make(map[string]*LogFile),
	}

	timer := time.NewTicker(time.Millisecond * 100)
	//timer := time.NewTicker(time.Second)
	go func() {
		for {
			select {
			case <-timer.C:
				//now := nowFunc()
				for _, file := range asyncLog.files {
					if file.sync.status != statusDoing {
						go file.flush()
					}
				}
			}
		}

	}()
}

func NewLogFile(filename string) *LogFile {
	asyncLog.Lock()
	if lf, ok := asyncLog.files[filename]; ok {
		asyncLog.Unlock()
		return lf
	}

	lf := &LogFile{
		filename: filename,
		flag:     StdFlag,
	}

	asyncLog.files[filename] = lf
	asyncLog.Unlock()

	// 默认按小时切割文件
	lf.logRotate.rotate = RotateHour

	// 默认开启缓存
	lf.cache.use = true

	// 日志写入概率，默认为1.1, 就是全部写入
	lf.probability = 1.1

	// TODO 同步的时间周期，缓存开启才有效
	lf.sync.duration = time.Second
	return lf
}

func (lf *LogFile) SetFlags(flag int) {
	lf.flag = flag
}

func (lf *LogFile) SetRotate(rotate LogRotate) {
	lf.logRotate.rotate = rotate
}

func (lf *LogFile) SetUseCache(useCache bool) {
	lf.cache.use = useCache
}

func (lf *LogFile) SetProbability(probability float32) {
	lf.probability = probability
}

// Write 写缓存
func (lf *LogFile) Write(msg string) error {
	if lf.flag == StdFlag {
		msg = nowFunc().Format(logTimeFormat) + " " + msg + newlineChar
	} else {
		msg = msg + newlineChar
	}

	if lf.cache.use {
		lf.appendCache(msg)
		return nil
	}

	return lf.directWrite([]byte(msg))
}

// WriteJson 写入json数据
func (lf *LogFile) WriteJson(data interface{}) error {
	if lf.probability < 1.0 && rand.Float32() > lf.probability {
		// 按照概率写入
		return nil
	}

	bts, err := json.Marshal(data)
	if err != nil {
		return err
	}

	if lf.cache.use {
		lf.appendCache(string(bts) + newlineChar)
		return nil
	}

	return lf.directWrite(bts)
}

//*********************** 以下是私有函数 ************************************

func (lf *LogFile) appendCache(msg string) {
	lf.cache.writeMutex.Lock()
	lf.cache.data = append(lf.cache.data, msg)
	lf.cache.writeMutex.Unlock()
}

// 同步缓存到文件中
func (lf *LogFile) flush() error {
	lf.writeMutex.Lock()
	defer lf.writeMutex.Unlock()

	lf.sync.status = statusDoing
	defer func() {
		lf.sync.status = statusDone
	}()

	// 写入log文件
	file, err := lf.openFileNoCache()
	if err != nil {
		panic(err)
	}
	defer file.Close()

	// 获取缓存数据
	lf.cache.writeMutex.Lock()
	cache := lf.cache.data
	lf.cache.data = make([]string, 0, cacheInitCap)
	lf.cache.writeMutex.Unlock()

	if len(cache) == 0 {
		return nil
	}

	_, err = file.WriteString(strings.Join(cache, ""))
	if err != nil {
		// 重试
		_, err = file.WriteString(strings.Join(cache, ""))
		if err != nil {
			panic(err)
		}
	}

	return nil
}

// 获取文件名的后缀
func (lf *LogFile) getFilenameSuffix() string {
	if lf.logRotate.rotate == RotateDate {
		return nowFunc().Format("20060102")
	}
	return nowFunc().Format("2006010215")
}

// 直接写入日志文件
func (lf *LogFile) directWrite(msg []byte) error {
	file, err := lf.openFile()
	//file, err := lf.openFileNoCache()
	if err != nil {
		panic(err)
	}
	defer file.Close()

	lf.writeMutex.Lock()
	_, err = file.Write(msg)
	lf.writeMutex.Unlock()

	return err
}

// 打开日志文件
func (lf *LogFile) openFile() (*os.File, error) {
	suffix := lf.getFilenameSuffix()

	lf.logRotate.mutex.Lock()
	defer lf.logRotate.mutex.Unlock()

	if suffix == lf.logRotate.suffix {
		return lf.logRotate.file, nil
	}

	logFilename := lf.filename + "." + suffix
	file, err := os.OpenFile(logFilename, fileFlag, fileOpenMode)
	if err != nil {
		// 重试
		file, err = os.OpenFile(logFilename, fileFlag, fileOpenMode)
		if err != nil {
			return file, err
		}
	}

	// 关闭旧的文件
	if lf.logRotate.file != nil {
		lf.logRotate.file.Close()
	}

	lf.logRotate.file = file
	lf.logRotate.suffix = suffix
	return file, nil
}

// 打开日志文件(不缓存句柄)
func (lf *LogFile) openFileNoCache() (*os.File, error) {
	logFilename := lf.filename + "." + lf.getFilenameSuffix()
	file, err := os.OpenFile(logFilename, fileFlag, fileOpenMode)
	if err != nil {
		// 重试
		file, err = os.OpenFile(logFilename, fileFlag, fileOpenMode)
		if err != nil {
			return file, err
		}
	}

	return file, nil
}
<commit_msg>fix open too many files<commit_after>package asyncLog

import (
	"encoding/json"
	"log"
	"math/rand"
	"os"
	"strings"
	"sync"
	"time"
)

type asyncLogType struct {
	files map[string]*LogFile // 下标是文件名

	// 避免并发new对象
	sync.Mutex
}

type LogFile struct {
	filename   string     // 原始文件名（包含完整目录）
	flag       int        // 默认为log.LstdFlags
	writeMutex sync.Mutex // 写文件时的锁

	// 同步设置
	sync struct {
		duration  time.Duration // 同步数据到文件的周期，默认为1秒
		beginTime time.Time     // 开始同步的时间，判断同步的耗时
		status    syncStatus    // 同步状态
	}

	// 日志的等级
	level Priority

	// 缓存
	cache struct {
		use  bool     // 是否使用缓存
		data []string // 缓存数据

		// 写cache时的互斥锁
		writeMutex sync.Mutex
	}

	// 文件切割
	logRotate struct {
		rotate LogRotate  // 默认按小时切割
		file   *os.File   // 文件操作对象
		suffix string     // 切割后的文件名后缀
		mutex  sync.Mutex // 文件名锁
	}

	// 日志写入概率
	probability float32
}

// log同步的状态
type syncStatus int

const (
	statusInit  syncStatus = iota // 初始状态
	statusDoing                   // 同步中
	statusDone                    // 同步已经完成
)

// 日志切割的方式
type LogRotate int

const (
	RotateHour LogRotate = iota // 按小时切割
	RotateDate                  // 按日期切割
)

const (
	// 写日志时前缀的时间格式
	// "2006-01-02T15:04:05Z07:00"
	logTimeFormat string = time.RFC3339

	// 文件写入mode
	fileOpenMode = 0666

	// 文件Flag
	fileFlag = os.O_WRONLY | os.O_CREATE | os.O_APPEND

	// 换行符
	newlineChar = "\n"

	// 缓存切片的初始容量
	cacheInitCap = 64
)

// 是否需要Flag信息
const (
	NoFlag  = 0
	StdFlag = log.LstdFlags
)

// 异步日志变量
var asyncLog *asyncLogType

var nowFunc = time.Now

func init() {
	asyncLog = &asyncLogType{
		files: make(map[string]*LogFile),
	}

	timer := time.NewTicker(time.Millisecond * 100)
	//timer := time.NewTicker(time.Second)
	go func() {
		for {
			select {
			case <-timer.C:
				//now := nowFunc()
				for _, file := range asyncLog.files {
					if file.sync.status != statusDoing {
						go file.flush()
					}
				}
			}
		}

	}()
}

func NewLogFile(filename string) *LogFile {
	asyncLog.Lock()
	if lf, ok := asyncLog.files[filename]; ok {
		asyncLog.Unlock()
		return lf
	}

	lf := &LogFile{
		filename: filename,
		flag:     StdFlag,
	}

	asyncLog.files[filename] = lf
	asyncLog.Unlock()

	// 默认按小时切割文件
	lf.logRotate.rotate = RotateHour

	// 默认开启缓存
	lf.cache.use = true

	// 日志写入概率，默认为1.1, 就是全部写入
	lf.probability = 1.1

	// TODO 同步的时间周期，缓存开启才有效
	lf.sync.duration = time.Second
	return lf
}

func (lf *LogFile) SetFlags(flag int) {
	lf.flag = flag
}

func (lf *LogFile) SetRotate(rotate LogRotate) {
	lf.logRotate.rotate = rotate
}

func (lf *LogFile) SetUseCache(useCache bool) {
	lf.cache.use = useCache
}

func (lf *LogFile) SetProbability(probability float32) {
	lf.probability = probability
}

// Write 写缓存
func (lf *LogFile) Write(msg string) error {
	if lf.flag == StdFlag {
		msg = nowFunc().Format(logTimeFormat) + " " + msg + newlineChar
	} else {
		msg = msg + newlineChar
	}

	if lf.cache.use {
		lf.appendCache(msg)
		return nil
	}

	return lf.directWrite([]byte(msg))
}

// WriteJson 写入json数据
func (lf *LogFile) WriteJson(data interface{}) error {
	if lf.probability < 1.0 && rand.Float32() > lf.probability {
		// 按照概率写入
		return nil
	}

	bts, err := json.Marshal(data)
	if err != nil {
		return err
	}

	if lf.cache.use {
		lf.appendCache(string(bts) + newlineChar)
		return nil
	}

	return lf.directWrite(bts)
}

//*********************** 以下是私有函数 ************************************

func (lf *LogFile) appendCache(msg string) {
	lf.cache.writeMutex.Lock()
	lf.cache.data = append(lf.cache.data, msg)
	lf.cache.writeMutex.Unlock()
}

// 同步缓存到文件中
func (lf *LogFile) flush() error {

	lf.sync.status = statusDoing
	defer func() {
		lf.sync.status = statusDone
	}()

	// 写入log文件
	file, err := lf.openFileNoCache()
	if err != nil {
		panic(err)
	}
	defer file.Close()

	// 获取缓存数据
	lf.cache.writeMutex.Lock()
	cache := lf.cache.data
	lf.cache.data = make([]string, 0, cacheInitCap)
	lf.cache.writeMutex.Unlock()

	if len(cache) == 0 {
		return nil
	}

	_, err = file.WriteString(strings.Join(cache, ""))
	if err != nil {
		// 重试
		_, err = file.WriteString(strings.Join(cache, ""))
		if err != nil {
			panic(err)
		}
	}

	return nil
}

// 获取文件名的后缀
func (lf *LogFile) getFilenameSuffix() string {
	if lf.logRotate.rotate == RotateDate {
		return nowFunc().Format("20060102")
	}
	return nowFunc().Format("2006010215")
}

// 直接写入日志文件
func (lf *LogFile) directWrite(msg []byte) error {
	file, err := lf.openFile()
	//file, err := lf.openFileNoCache()
	if err != nil {
		panic(err)
	}
	defer file.Close()

	lf.writeMutex.Lock()
	_, err = file.Write(msg)
	lf.writeMutex.Unlock()

	return err
}

// 打开日志文件
func (lf *LogFile) openFile() (*os.File, error) {
	suffix := lf.getFilenameSuffix()

	lf.logRotate.mutex.Lock()
	defer lf.logRotate.mutex.Unlock()

	if suffix == lf.logRotate.suffix {
		return lf.logRotate.file, nil
	}

	logFilename := lf.filename + "." + suffix
	file, err := os.OpenFile(logFilename, fileFlag, fileOpenMode)
	if err != nil {
		// 重试
		file, err = os.OpenFile(logFilename, fileFlag, fileOpenMode)
		if err != nil {
			return file, err
		}
	}

	// 关闭旧的文件
	if lf.logRotate.file != nil {
		lf.logRotate.file.Close()
	}

	lf.logRotate.file = file
	lf.logRotate.suffix = suffix
	return file, nil
}

// 打开日志文件(不缓存句柄)
func (lf *LogFile) openFileNoCache() (*os.File, error) {
	logFilename := lf.filename + "." + lf.getFilenameSuffix()

	lf.writeMutex.Lock()
	defer lf.writeMutex.Unlock()
	file, err := os.OpenFile(logFilename, fileFlag, fileOpenMode)
	if err != nil {
		// 重试
		file, err = os.OpenFile(logFilename, fileFlag, fileOpenMode)
		if err != nil {
			return file, err
		}
	}

	return file, nil
}
<|endoftext|>