<commit_before>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tag

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"strings"
	"unicode/utf16"
)

// when the frame is not encoded, add a 0 at the start
func readTFrame(b []byte, encoded bool) (string, error) {
	if !encoded {
		b = append([]byte{0}, b[0:]...)
	}
	txt, err := parseText(b)
	if err != nil {
		return "", err
	}
	return strings.Join(strings.Split(txt, string([]byte{0})), ""), nil
}

func parseText(b []byte) (string, error) {
	if len(b) == 0 {
		return "", nil
	}
	return decodeText(b[0], b[1:])
}

func decodeText(enc byte, b []byte) (string, error) {
	if len(b) == 0 {
		return "", nil
	}

	switch enc {
	case 0: // ISO-8859-1
		return decodeISO8859(b), nil

	case 1: // UTF-16 with byte order marker
		if len(b) == 1 {
			return "", nil
		}
		return decodeUTF16WithBOM(b)

	case 2: // UTF-16 without byte order (assuming BigEndian)
		if len(b) == 1 {
			return "", nil
		}
		return decodeUTF16(b, binary.BigEndian), nil

	case 3: // UTF-8
		return string(b), nil

	default:
		return "", fmt.Errorf("invalid encoding byte %x", enc)
	}
}

func encodingDelim(enc byte) ([]byte, error) {
	switch enc {
	case 0, 3: // see decodeText above
		return []byte{0}, nil
	case 1, 2: // see decodeText above
		return []byte{0, 0}, nil
	default:
		return nil, fmt.Errorf("invalid encoding byte %x", enc)
	}
}

func dataSplit(b []byte, enc byte) ([][]byte, error) {
	delim, err := encodingDelim(enc)
	if err != nil {
		return nil, err
	}
	result := bytes.SplitN(b, delim, 2)

	if len(result) <= 1 {
		return result, nil
	}

	if result[1][0] == 0 {
		// there was a double (or triple) 0 and we cut too early
		result[0] = append(result[0], make([]byte, 0)...)
		result[1] = result[1][1:]
	}

	return result, nil
}

func decodeISO8859(b []byte) string {
	r := make([]rune, len(b))
	for i, x := range b {
		r[i] = rune(x)
	}
	return string(r)
}

func decodeUTF16WithBOM(b []byte) (string, error) {
	var bo binary.ByteOrder
	switch {
	case b[0] == 0xFE && b[1] == 0xFF:
		bo = binary.BigEndian

	case b[0] == 0xFF && b[1] == 0xFE:
		bo = binary.LittleEndian

	default:
		return "", fmt.Errorf("invalid byte order marker %x %x", b[0], b[1])
	}
	return decodeUTF16(b[2:], bo), nil
}

func decodeUTF16(b []byte, bo binary.ByteOrder) string {
	s := make([]uint16, 0, len(b)/2)
	for i := 0; i < len(b); i += 2 {
		s = append(s, bo.Uint16(b[i:i+2]))
	}
	return string(utf16.Decode(s))
}

// Comm is a type used in COMM, UFID, TXXX, WXXX and USLT tag.
// It's a text with a description and a specified language
// For WXXX, TXXX and UFID, we don't set a Language

type Comm struct {
	Language    string
	Description string
	Text        string
}

// String returns a string representation of the underlying Comm instance.
func (t Comm) String() string {
	if t.Language != "" {
		return fmt.Sprintf("Text{Lang: '%v', Description: '%v', %v lines}",
			t.Language, t.Description, strings.Count(t.Text, "\n"))
	}
	return fmt.Sprintf("Text{Description: '%v', %v}", t.Description, t.Text)
}

// IDv2.{3,4}
// -- Header
// <Header for 'Unsynchronised lyrics/text transcription', ID: "USLT">
// <Header for 'Comment', ID: "COMM">
// -- readTextWithDescrFrame(data, true, true)
// Text encoding       $xx
// Language            $xx xx xx
// Content descriptor  <text string according to encoding> $00 (00)
// Lyrics/text         <full text string according to encoding>
// -- Header
// <Header for         'User defined text information frame', ID: "TXXX">
// <Header for         'User defined URL link frame', ID: "WXXX">
// -- readTextWithDescrFrame(data, false, <isDataEncoded>)
// Text encoding       $xx
// Description         <text string according to encoding> $00 (00)
// Value               <text string according to encoding>
func readTextWithDescrFrame(b []byte, hasLang bool, encoded bool) (*Comm, error) {
	var descTextSplit [][]byte
	var lang string
	var err error
	enc := b[0]

	if hasLang {
		lang = string(b[1:4])
		descTextSplit, err = dataSplit(b[4:], enc)
	} else {
		lang = ""
		descTextSplit, err = dataSplit(b[1:], enc)
	}
	if err != nil {
		return nil, err
	}

	desc, err := decodeText(enc, descTextSplit[0])
	if err != nil {
		return nil, fmt.Errorf("error decoding tag description text: %v", err)
	}

	if !encoded {
		enc = byte(0)
	}

	text, err := decodeText(enc, descTextSplit[1])
	if err != nil {
		return nil, fmt.Errorf("error decoding tag text: %v", err)
	}

	return &Comm{
		Language:    lang,
		Description: desc,
		Text:        text,
	}, nil
}

// UFID is composed of a provider (frequently a URL and a binary identifier)
// The identifier can be a text (Musicbrainz use texts, but not necessary)
type UFID struct {
	Provider   string
	Identifier []byte
}

func (u UFID) String() string {
	return fmt.Sprintf("%v (%v)", u.Provider, string(u.Identifier))
}

func readUFID(b []byte) (*UFID, error) {
	result := bytes.SplitN(b, []byte{0}, 2)

	return &UFID{
		Provider:   string(result[0]),
		Identifier: result[1],
	}, nil
}

var pictureTypes = map[byte]string{
	0x00: "Other",
	0x01: "32x32 pixels 'file icon' (PNG only)",
	0x02: "Other file icon",
	0x03: "Cover (front)",
	0x04: "Cover (back)",
	0x05: "Leaflet page",
	0x06: "Media (e.g. lable side of CD)",
	0x07: "Lead artist/lead performer/soloist",
	0x08: "Artist/performer",
	0x09: "Conductor",
	0x0A: "Band/Orchestra",
	0x0B: "Composer",
	0x0C: "Lyricist/text writer",
	0x0D: "Recording Location",
	0x0E: "During recording",
	0x0F: "During performance",
	0x10: "Movie/video screen capture",
	0x11: "A bright coloured fish",
	0x12: "Illustration",
	0x13: "Band/artist logotype",
	0x14: "Publisher/Studio logotype",
}

// Picture is a type which represents an attached picture extracted from metadata.
type Picture struct {
	Ext         string // Extension of the picture file.
	MIMEType    string // MIMEType of the picture.
	Type        string // Type of the picture (see pictureTypes).
	Description string // Description.
	Data        []byte // Raw picture data.
}

// String returns a string representation of the underlying Picture instance.
func (p Picture) String() string {
	return fmt.Sprintf("Picture{Ext: %v, MIMEType: %v, Type: %v, Description: %v, Data.Size: %v}",
		p.Ext, p.MIMEType, p.Type, p.Description, len(p.Data))
}

// IDv2.2
// -- Header
// Attached picture   "PIC"
// Frame size         $xx xx xx
// -- readPICFrame
// Text encoding      $xx
// Image format       $xx xx xx
// Picture type       $xx
// Description        <textstring> $00 (00)
// Picture data       <binary data>
func readPICFrame(b []byte) (*Picture, error) {
	enc := b[0]
	ext := string(b[1:4])
	picType := b[4]

	descDataSplit, err := dataSplit(b[5:], enc)
	if err != nil {
		return nil, err
	}
	desc, err := decodeText(enc, descDataSplit[0])
	if err != nil {
		return nil, fmt.Errorf("error decoding PIC description text: %v", err)
	}

	var mimeType string
	switch ext {
	case "jpeg", "jpg":
		mimeType = "image/jpeg"
	case "png":
		mimeType = "image/png"
	}

	return &Picture{
		Ext:         ext,
		MIMEType:    mimeType,
		Type:        pictureTypes[picType],
		Description: desc,
		Data:        descDataSplit[1],
	}, nil
}

// IDv2.{3,4}
// -- Header
// <Header for 'Attached picture', ID: "APIC">
// -- readAPICFrame
// Text encoding   $xx
// MIME type       <text string> $00
// Picture type    $xx
// Description     <text string according to encoding> $00 (00)
// Picture data    <binary data>
func readAPICFrame(b []byte) (*Picture, error) {
	enc := b[0]
	mimeDataSplit := bytes.SplitN(b[1:], []byte{0}, 2)
	mimeType := string(mimeDataSplit[0])

	b = mimeDataSplit[1]
	picType := b[0]

	descDataSplit, err := dataSplit(b[1:], enc)
	if err != nil {
		return nil, err
	}
	desc, err := decodeText(enc, descDataSplit[0])
	if err != nil {
		return nil, fmt.Errorf("error decoding APIC description text: %v", err)
	}

	var ext string
	switch mimeType {
	case "image/jpeg":
		ext = "jpg"
	case "image/png":
		ext = "png"
	}

	return &Picture{
		Ext:         ext,
		MIMEType:    mimeType,
		Type:        pictureTypes[picType],
		Description: desc,
		Data:        descDataSplit[1],
	}, nil
}
<commit_msg>Add check for size of provider/identifier split<commit_after>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tag

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"strings"
	"unicode/utf16"
)

// when the frame is not encoded, add a 0 at the start
func readTFrame(b []byte, encoded bool) (string, error) {
	if !encoded {
		b = append([]byte{0}, b[0:]...)
	}
	txt, err := parseText(b)
	if err != nil {
		return "", err
	}
	return strings.Join(strings.Split(txt, string([]byte{0})), ""), nil
}

func parseText(b []byte) (string, error) {
	if len(b) == 0 {
		return "", nil
	}
	return decodeText(b[0], b[1:])
}

func decodeText(enc byte, b []byte) (string, error) {
	if len(b) == 0 {
		return "", nil
	}

	switch enc {
	case 0: // ISO-8859-1
		return decodeISO8859(b), nil

	case 1: // UTF-16 with byte order marker
		if len(b) == 1 {
			return "", nil
		}
		return decodeUTF16WithBOM(b)

	case 2: // UTF-16 without byte order (assuming BigEndian)
		if len(b) == 1 {
			return "", nil
		}
		return decodeUTF16(b, binary.BigEndian), nil

	case 3: // UTF-8
		return string(b), nil

	default:
		return "", fmt.Errorf("invalid encoding byte %x", enc)
	}
}

func encodingDelim(enc byte) ([]byte, error) {
	switch enc {
	case 0, 3: // see decodeText above
		return []byte{0}, nil
	case 1, 2: // see decodeText above
		return []byte{0, 0}, nil
	default:
		return nil, fmt.Errorf("invalid encoding byte %x", enc)
	}
}

func dataSplit(b []byte, enc byte) ([][]byte, error) {
	delim, err := encodingDelim(enc)
	if err != nil {
		return nil, err
	}
	result := bytes.SplitN(b, delim, 2)

	if len(result) <= 1 {
		return result, nil
	}

	if result[1][0] == 0 {
		// there was a double (or triple) 0 and we cut too early
		result[0] = append(result[0], make([]byte, 0)...)
		result[1] = result[1][1:]
	}

	return result, nil
}

func decodeISO8859(b []byte) string {
	r := make([]rune, len(b))
	for i, x := range b {
		r[i] = rune(x)
	}
	return string(r)
}

func decodeUTF16WithBOM(b []byte) (string, error) {
	var bo binary.ByteOrder
	switch {
	case b[0] == 0xFE && b[1] == 0xFF:
		bo = binary.BigEndian

	case b[0] == 0xFF && b[1] == 0xFE:
		bo = binary.LittleEndian

	default:
		return "", fmt.Errorf("invalid byte order marker %x %x", b[0], b[1])
	}
	return decodeUTF16(b[2:], bo), nil
}

func decodeUTF16(b []byte, bo binary.ByteOrder) string {
	s := make([]uint16, 0, len(b)/2)
	for i := 0; i < len(b); i += 2 {
		s = append(s, bo.Uint16(b[i:i+2]))
	}
	return string(utf16.Decode(s))
}

// Comm is a type used in COMM, UFID, TXXX, WXXX and USLT tag.
// It's a text with a description and a specified language
// For WXXX, TXXX and UFID, we don't set a Language

type Comm struct {
	Language    string
	Description string
	Text        string
}

// String returns a string representation of the underlying Comm instance.
func (t Comm) String() string {
	if t.Language != "" {
		return fmt.Sprintf("Text{Lang: '%v', Description: '%v', %v lines}",
			t.Language, t.Description, strings.Count(t.Text, "\n"))
	}
	return fmt.Sprintf("Text{Description: '%v', %v}", t.Description, t.Text)
}

// IDv2.{3,4}
// -- Header
// <Header for 'Unsynchronised lyrics/text transcription', ID: "USLT">
// <Header for 'Comment', ID: "COMM">
// -- readTextWithDescrFrame(data, true, true)
// Text encoding       $xx
// Language            $xx xx xx
// Content descriptor  <text string according to encoding> $00 (00)
// Lyrics/text         <full text string according to encoding>
// -- Header
// <Header for         'User defined text information frame', ID: "TXXX">
// <Header for         'User defined URL link frame', ID: "WXXX">
// -- readTextWithDescrFrame(data, false, <isDataEncoded>)
// Text encoding       $xx
// Description         <text string according to encoding> $00 (00)
// Value               <text string according to encoding>
func readTextWithDescrFrame(b []byte, hasLang bool, encoded bool) (*Comm, error) {
	var descTextSplit [][]byte
	var lang string
	var err error
	enc := b[0]

	if hasLang {
		lang = string(b[1:4])
		descTextSplit, err = dataSplit(b[4:], enc)
	} else {
		lang = ""
		descTextSplit, err = dataSplit(b[1:], enc)
	}
	if err != nil {
		return nil, err
	}

	desc, err := decodeText(enc, descTextSplit[0])
	if err != nil {
		return nil, fmt.Errorf("error decoding tag description text: %v", err)
	}

	if !encoded {
		enc = byte(0)
	}

	text, err := decodeText(enc, descTextSplit[1])
	if err != nil {
		return nil, fmt.Errorf("error decoding tag text: %v", err)
	}

	return &Comm{
		Language:    lang,
		Description: desc,
		Text:        text,
	}, nil
}

// UFID is composed of a provider (frequently a URL and a binary identifier)
// The identifier can be a text (Musicbrainz use texts, but not necessary)
type UFID struct {
	Provider   string
	Identifier []byte
}

func (u UFID) String() string {
	return fmt.Sprintf("%v (%v)", u.Provider, string(u.Identifier))
}

func readUFID(b []byte) (*UFID, error) {
	result := bytes.SplitN(b, []byte{0}, 2)
	if len(result) != 2 {
		return nil, errors.New("expected to split UFID data into 2 pieces")
	}

	return &UFID{
		Provider:   string(result[0]),
		Identifier: result[1],
	}, nil
}

var pictureTypes = map[byte]string{
	0x00: "Other",
	0x01: "32x32 pixels 'file icon' (PNG only)",
	0x02: "Other file icon",
	0x03: "Cover (front)",
	0x04: "Cover (back)",
	0x05: "Leaflet page",
	0x06: "Media (e.g. lable side of CD)",
	0x07: "Lead artist/lead performer/soloist",
	0x08: "Artist/performer",
	0x09: "Conductor",
	0x0A: "Band/Orchestra",
	0x0B: "Composer",
	0x0C: "Lyricist/text writer",
	0x0D: "Recording Location",
	0x0E: "During recording",
	0x0F: "During performance",
	0x10: "Movie/video screen capture",
	0x11: "A bright coloured fish",
	0x12: "Illustration",
	0x13: "Band/artist logotype",
	0x14: "Publisher/Studio logotype",
}

// Picture is a type which represents an attached picture extracted from metadata.
type Picture struct {
	Ext         string // Extension of the picture file.
	MIMEType    string // MIMEType of the picture.
	Type        string // Type of the picture (see pictureTypes).
	Description string // Description.
	Data        []byte // Raw picture data.
}

// String returns a string representation of the underlying Picture instance.
func (p Picture) String() string {
	return fmt.Sprintf("Picture{Ext: %v, MIMEType: %v, Type: %v, Description: %v, Data.Size: %v}",
		p.Ext, p.MIMEType, p.Type, p.Description, len(p.Data))
}

// IDv2.2
// -- Header
// Attached picture   "PIC"
// Frame size         $xx xx xx
// -- readPICFrame
// Text encoding      $xx
// Image format       $xx xx xx
// Picture type       $xx
// Description        <textstring> $00 (00)
// Picture data       <binary data>
func readPICFrame(b []byte) (*Picture, error) {
	enc := b[0]
	ext := string(b[1:4])
	picType := b[4]

	descDataSplit, err := dataSplit(b[5:], enc)
	if err != nil {
		return nil, err
	}
	desc, err := decodeText(enc, descDataSplit[0])
	if err != nil {
		return nil, fmt.Errorf("error decoding PIC description text: %v", err)
	}

	var mimeType string
	switch ext {
	case "jpeg", "jpg":
		mimeType = "image/jpeg"
	case "png":
		mimeType = "image/png"
	}

	return &Picture{
		Ext:         ext,
		MIMEType:    mimeType,
		Type:        pictureTypes[picType],
		Description: desc,
		Data:        descDataSplit[1],
	}, nil
}

// IDv2.{3,4}
// -- Header
// <Header for 'Attached picture', ID: "APIC">
// -- readAPICFrame
// Text encoding   $xx
// MIME type       <text string> $00
// Picture type    $xx
// Description     <text string according to encoding> $00 (00)
// Picture data    <binary data>
func readAPICFrame(b []byte) (*Picture, error) {
	enc := b[0]
	mimeDataSplit := bytes.SplitN(b[1:], []byte{0}, 2)
	mimeType := string(mimeDataSplit[0])

	b = mimeDataSplit[1]
	picType := b[0]

	descDataSplit, err := dataSplit(b[1:], enc)
	if err != nil {
		return nil, err
	}
	desc, err := decodeText(enc, descDataSplit[0])
	if err != nil {
		return nil, fmt.Errorf("error decoding APIC description text: %v", err)
	}

	var ext string
	switch mimeType {
	case "image/jpeg":
		ext = "jpg"
	case "image/png":
		ext = "png"
	}

	return &Picture{
		Ext:         ext,
		MIMEType:    mimeType,
		Type:        pictureTypes[picType],
		Description: desc,
		Data:        descDataSplit[1],
	}, nil
}
<|endoftext|>


<commit_before>package recorder

import (
	"os"
	"os/exec"
	//"strconv"
)

var FfmpegPath = "ffmpeg"

func NewFfmpeg(r *Recorder) (*Command, error) {

	// Check if custom phantomjs bin path
	if path := os.Getenv("FFMPEG_BIN"); path != "" {
		FfmpegPath = path
	}

	cmd := exec.Command(FfmpegPath, "-r", "10", "-y", "-c:v", "png", "-f", "image2pipe", "-i", "-", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-movflags", "+faststart", r.OutputFile)

	// Map various pipes
	inPipe, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	outPipe, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	errPipe, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}

	ffm := &Command{
		Cmd:    cmd,
		In:     inPipe,
		Out:    outPipe,
		Errout: errPipe,
	}

	return ffm, nil

}
<commit_msg>write temp file to tmp dir<commit_after>package recorder

import (
	"os"
	"os/exec"
	//"strconv"
)

var FfmpegPath = "ffmpeg"

func NewFfmpeg(r *Recorder) (*Command, error) {

	// Check if custom phantomjs bin path
	if path := os.Getenv("FFMPEG_BIN"); path != "" {
		FfmpegPath = path
	}

	cmd := exec.Command(FfmpegPath, "-r", "10", "-y", "-c:v", "png", "-f", "image2pipe", "-i", "-", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-movflags", "+faststart", "/tmp"+r.OutputFile)

	// Map various pipes
	inPipe, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	outPipe, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	errPipe, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}

	ffm := &Command{
		Cmd:    cmd,
		In:     inPipe,
		Out:    outPipe,
		Errout: errPipe,
	}

	return ffm, nil

}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package file

import "testing"

// Open this file itself and verify that the first few characters are
// as expected.
func TestRead(t *testing.T) {
	f := Fopen("file_test.go", "r")
	if f == nil {
		t.Fatal("fopen failed")
	}
	if Fgetc(f) != '/' || Fgetc(f) != '/' || Fgetc(f) != ' ' || Fgetc(f) != 'C' {
		t.Error("read unexpected characters")
	}
	if Fclose(f) != 0 {
		t.Error("fclose failed")
	}
}
<commit_msg>misc/swig/stdio: fix broken nil pointer test<commit_after>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package file

import "testing"

// Open this file itself and verify that the first few characters are
// as expected.
func TestRead(t *testing.T) {
	f := Fopen("file_test.go", "r")
	if f.Swigcptr() == 0 {
		t.Fatal("fopen failed")
	}
	if Fgetc(f) != '/' || Fgetc(f) != '/' || Fgetc(f) != ' ' || Fgetc(f) != 'C' {
		t.Error("read unexpected characters")
	}
	if Fclose(f) != 0 {
		t.Error("fclose failed")
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The Zombie Zen Log Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// SPDX-License-Identifier: BSD-3-Clause

package log

import (
	"context"
	"fmt"
	"runtime"
	"time"
)

// Infof writes an info message to the default Logger.  Its arguments are handled in the manner of fmt.Sprintf.
func Infof(ctx context.Context, format string, args ...interface{}) {
	if false {
		// Enable printf checking in go vet.
		_ = fmt.Sprintf(format, args...)
	}
	logf(ctx, Default(), Info, format, args)
}

// Debugf writes a debug message to the default Logger.  Its arguments are handled in the manner of fmt.Sprintf.
func Debugf(ctx context.Context, format string, args ...interface{}) {
	if false {
		// Enable printf checking in go vet.
		_ = fmt.Sprintf(format, args...)
	}
	logf(ctx, Default(), Debug, format, args)
}

// Warnf writes a warning message to the default Logger.  Its arguments are handled in the manner of fmt.Sprintf.
func Warnf(ctx context.Context, format string, args ...interface{}) {
	if false {
		// Enable printf checking in go vet.
		_ = fmt.Sprintf(format, args...)
	}
	logf(ctx, Default(), Warn, format, args)
}

// Errorf writes an error message to the default Logger.  Its arguments are handled in the manner of fmt.Sprintf.
func Errorf(ctx context.Context, format string, args ...interface{}) {
	if false {
		// Enable printf checking in go vet.
		_ = fmt.Sprintf(format, args...)
	}
	logf(ctx, Default(), Error, format, args)
}

// Logf writes a message to a Logger.  Its arguments are handled in the manner of fmt.Sprintf.
func Logf(ctx context.Context, logger Logger, level Level, format string, args ...interface{}) {
	if false {
		// Enable printf checking in go vet.
		_ = fmt.Sprintf(format, args...)
	}
	logf(ctx, logger, level, format, args)
}

func logf(ctx context.Context, logger Logger, level Level, format string, args []interface{}) {
	ent := Entry{Time: time.Now(), Level: level}
	if _, file, line, ok := runtime.Caller(2); ok {
		ent.File = file
		ent.Line = line
	}
	if !logger.LogEnabled(ent) {
		return
	}
	ent.Msg = fmt.Sprintf(format, args...)
	if n := len(ent.Msg); n > 0 && ent.Msg[n-1] == '\n' {
		ent.Msg = ent.Msg[:n-1]
	}
	logger.Log(ctx, ent)
}
<commit_msg>log: add IsEnabled function<commit_after>// Copyright 2017 The Zombie Zen Log Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// SPDX-License-Identifier: BSD-3-Clause

package log

import (
	"context"
	"fmt"
	"runtime"
	"time"
)

// Infof writes an info message to the default Logger.  Its arguments are handled in the manner of fmt.Sprintf.
func Infof(ctx context.Context, format string, args ...interface{}) {
	if false {
		// Enable printf checking in go vet.
		_ = fmt.Sprintf(format, args...)
	}
	logf(ctx, Default(), Info, format, args)
}

// Debugf writes a debug message to the default Logger.  Its arguments are handled in the manner of fmt.Sprintf.
func Debugf(ctx context.Context, format string, args ...interface{}) {
	if false {
		// Enable printf checking in go vet.
		_ = fmt.Sprintf(format, args...)
	}
	logf(ctx, Default(), Debug, format, args)
}

// Warnf writes a warning message to the default Logger.  Its arguments are handled in the manner of fmt.Sprintf.
func Warnf(ctx context.Context, format string, args ...interface{}) {
	if false {
		// Enable printf checking in go vet.
		_ = fmt.Sprintf(format, args...)
	}
	logf(ctx, Default(), Warn, format, args)
}

// Errorf writes an error message to the default Logger.  Its arguments are handled in the manner of fmt.Sprintf.
func Errorf(ctx context.Context, format string, args ...interface{}) {
	if false {
		// Enable printf checking in go vet.
		_ = fmt.Sprintf(format, args...)
	}
	logf(ctx, Default(), Error, format, args)
}

// Logf writes a message to a Logger.  Its arguments are handled in the manner of fmt.Sprintf.
func Logf(ctx context.Context, logger Logger, level Level, format string, args ...interface{}) {
	if false {
		// Enable printf checking in go vet.
		_ = fmt.Sprintf(format, args...)
	}
	logf(ctx, logger, level, format, args)
}

func logf(ctx context.Context, logger Logger, level Level, format string, args []interface{}) {
	ent := Entry{Time: time.Now(), Level: level}
	if _, file, line, ok := runtime.Caller(2); ok {
		ent.File = file
		ent.Line = line
	}
	if !logger.LogEnabled(ent) {
		return
	}
	ent.Msg = fmt.Sprintf(format, args...)
	if n := len(ent.Msg); n > 0 && ent.Msg[n-1] == '\n' {
		ent.Msg = ent.Msg[:n-1]
	}
	logger.Log(ctx, ent)
}

// IsEnabled reports false if the default logger will no-op for logs at the given level.
func IsEnabled(level Level) bool {
	ent := Entry{Time: time.Now(), Level: level}
	if _, file, line, ok := runtime.Caller(1); ok {
		ent.File = file
		ent.Line = line
	}
	return Default().LogEnabled(ent)
}
<|endoftext|>


<commit_before>package fridge

import (
	"github.com/shomali11/fridge/item"
	"github.com/shomali11/xredis"
	"time"
)

const (
	// Fresh is when an item has not passed its "Best By" duration
	Fresh = "FRESH"

	// Cold is when an item has passed its "Best By" duration but not its "Use By" one
	Cold = "COLD"

	// Expired is when an item has passed its "Use By" duration
	Expired = "EXPIRED"

	// NotFound is when an item was not found due to being removed or was never stored before
	NotFound = "NOT_FOUND"

	// Refresh is when an item was restocked with a fresher one
	Refresh = "REFRESH"

	// OutOfStock is when an item needs restocking, but no restocking function was provided
	OutOfStock = "OUT_OF_STOCK"

	// Unchanged is when the restocked item is not different from the version in the cache
	Unchanged = "UNCHANGED"
)

const (
	empty = ""
)

// NewClient returns a client using an xredis client
func NewClient(xredisClient *xredis.Client, options ...ConfigOption) *Client {
	client := &Client{
		config:   newConfig(options...),
		itemDao:  item.NewDao(xredisClient),
		eventBus: newEventBus(),
	}
	return client
}

// Client fridge client
type Client struct {
	config   *Config
	itemDao  *item.Dao
	eventBus *EventBus
}

// Put an item
func (c *Client) Put(key string, value string, options ...item.ConfigOption) error {
	itemConfig := newItemConfig(c.config, options...)
	err := c.itemDao.SetConfig(key, itemConfig)
	if err != nil {
		return err
	}

	err = c.itemDao.Set(key, value, itemConfig.GetUseByInSeconds())
	if err != nil {
		return err
	}
	return nil
}

// Get an item
func (c *Client) Get(key string, options ...item.QueryConfigOption) (string, bool, error) {
	queryConfig := newQueryConfig(options...)
	restock := queryConfig.Restock

	itemConfig, found, err := c.itemDao.GetConfig(key)
	if err != nil {
		return empty, false, err
	}

	if !found {
		itemConfig = c.newDefaultItemConfig()
	}

	cachedValue, found, err := c.itemDao.Get(key)
	if err != nil {
		return empty, false, err
	}

	if !found {
		if itemConfig.Timestamp.IsZero() {
			go c.publish(key, NotFound)
		} else {
			go c.publish(key, Expired)
		}
		return c.restock(key, restock)
	}

	now := time.Now().UTC()
	if now.Before(itemConfig.Timestamp.Add(itemConfig.BestBy)) {
		go c.publish(key, Fresh)
		return cachedValue, true, nil
	}

	if now.Before(itemConfig.Timestamp.Add(itemConfig.UseBy)) {
		go c.publish(key, Cold)
		go c.restockAndCompare(key, cachedValue, restock)
		return cachedValue, true, nil
	}

	go c.publish(key, Expired)
	return c.restockAndCompare(key, cachedValue, restock)
}

// Remove an item
func (c *Client) Remove(key string) error {
	return c.itemDao.Remove(key)
}

// Ping pings redis
func (c *Client) Ping() error {
	return c.itemDao.Ping()
}

// Close closes resources
func (c *Client) Close() error {
	return c.itemDao.Close()
}

// HandleEvent overrides the default handleEvent callback
func (c *Client) HandleEvent(handleEvent func(event *Event)) {
	c.eventBus.HandleEvent(handleEvent)
}

func (c *Client) publish(key string, eventType string) {
	c.eventBus.Publish(key, eventType)
}

func (c *Client) restockAndCompare(key string, cachedValue string, callback func() (string, error)) (string, bool, error) {
	newValue, found, err := c.restock(key, callback)
	if err != nil || !found {
		return empty, found, err
	}

	if newValue == cachedValue {
		go c.publish(key, Unchanged)
	}
	return newValue, true, nil
}

func (c *Client) restock(key string, callback func() (string, error)) (string, bool, error) {
	if callback == nil {
		go c.publish(key, OutOfStock)
		return empty, false, nil
	}

	result, err := callback()
	if err != nil {
		return empty, false, err
	}

	go c.publish(key, Refresh)

	err = c.Put(key, result)
	if err != nil {
		return empty, false, err
	}
	return result, true, nil
}

func (c *Client) newDefaultItemConfig() *item.Config {
	return &item.Config{BestBy: c.config.defaultBestBy, UseBy: c.config.defaultUseBy}
}

func newItemConfig(config *Config, options ...item.ConfigOption) *item.Config {
	itemConfig := &item.Config{BestBy: config.defaultBestBy, UseBy: config.defaultUseBy}
	for _, option := range options {
		option(itemConfig)
	}
	return itemConfig
}

func newQueryConfig(options ...item.QueryConfigOption) *item.QueryConfig {
	queryConfig := &item.QueryConfig{}
	for _, option := range options {
		option(queryConfig)
	}
	return queryConfig
}
<commit_msg>Added validation<commit_after>package fridge

import (
	"errors"
	"github.com/shomali11/fridge/item"
	"github.com/shomali11/xredis"
	"time"
)

const (
	// Fresh is when an item has not passed its "Best By" duration
	Fresh = "FRESH"

	// Cold is when an item has passed its "Best By" duration but not its "Use By" one
	Cold = "COLD"

	// Expired is when an item has passed its "Use By" duration
	Expired = "EXPIRED"

	// NotFound is when an item was not found due to being removed or was never stored before
	NotFound = "NOT_FOUND"

	// Refresh is when an item was restocked with a fresher one
	Refresh = "REFRESH"

	// OutOfStock is when an item needs restocking, but no restocking function was provided
	OutOfStock = "OUT_OF_STOCK"

	// Unchanged is when the restocked item is not different from the version in the cache
	Unchanged = "UNCHANGED"
)

const (
	empty                 = ""
	invalidDurationsError = "Invalid 'best by' and 'use by' durations"
)

// NewClient returns a client using an xredis client
func NewClient(xredisClient *xredis.Client, options ...ConfigOption) *Client {
	client := &Client{
		config:   newConfig(options...),
		itemDao:  item.NewDao(xredisClient),
		eventBus: newEventBus(),
	}
	return client
}

// Client fridge client
type Client struct {
	config   *Config
	itemDao  *item.Dao
	eventBus *EventBus
}

// Put an item
func (c *Client) Put(key string, value string, options ...item.ConfigOption) error {
	itemConfig := newItemConfig(c.config, options...)
	if itemConfig.BestBy > itemConfig.UseBy {
		return errors.New(invalidDurationsError)
	}

	err := c.itemDao.SetConfig(key, itemConfig)
	if err != nil {
		return err
	}

	err = c.itemDao.Set(key, value, itemConfig.GetUseByInSeconds())
	if err != nil {
		return err
	}
	return nil
}

// Get an item
func (c *Client) Get(key string, options ...item.QueryConfigOption) (string, bool, error) {
	queryConfig := newQueryConfig(options...)
	restock := queryConfig.Restock

	itemConfig, found, err := c.itemDao.GetConfig(key)
	if err != nil {
		return empty, false, err
	}

	if !found {
		itemConfig = c.newDefaultItemConfig()
	}

	cachedValue, found, err := c.itemDao.Get(key)
	if err != nil {
		return empty, false, err
	}

	if !found {
		if itemConfig.Timestamp.IsZero() {
			go c.publish(key, NotFound)
		} else {
			go c.publish(key, Expired)
		}
		return c.restock(key, restock)
	}

	now := time.Now().UTC()
	if now.Before(itemConfig.Timestamp.Add(itemConfig.BestBy)) {
		go c.publish(key, Fresh)
		return cachedValue, true, nil
	}

	if now.Before(itemConfig.Timestamp.Add(itemConfig.UseBy)) {
		go c.publish(key, Cold)
		go c.restockAndCompare(key, cachedValue, restock)
		return cachedValue, true, nil
	}

	go c.publish(key, Expired)
	return c.restockAndCompare(key, cachedValue, restock)
}

// Remove an item
func (c *Client) Remove(key string) error {
	return c.itemDao.Remove(key)
}

// Ping pings redis
func (c *Client) Ping() error {
	return c.itemDao.Ping()
}

// Close closes resources
func (c *Client) Close() error {
	return c.itemDao.Close()
}

// HandleEvent overrides the default handleEvent callback
func (c *Client) HandleEvent(handleEvent func(event *Event)) {
	c.eventBus.HandleEvent(handleEvent)
}

func (c *Client) publish(key string, eventType string) {
	c.eventBus.Publish(key, eventType)
}

func (c *Client) restockAndCompare(key string, cachedValue string, callback func() (string, error)) (string, bool, error) {
	newValue, found, err := c.restock(key, callback)
	if err != nil || !found {
		return empty, found, err
	}

	if newValue == cachedValue {
		go c.publish(key, Unchanged)
	}
	return newValue, true, nil
}

func (c *Client) restock(key string, callback func() (string, error)) (string, bool, error) {
	if callback == nil {
		go c.publish(key, OutOfStock)
		return empty, false, nil
	}

	result, err := callback()
	if err != nil {
		return empty, false, err
	}

	go c.publish(key, Refresh)

	err = c.Put(key, result)
	if err != nil {
		return empty, false, err
	}
	return result, true, nil
}

func (c *Client) newDefaultItemConfig() *item.Config {
	return &item.Config{BestBy: c.config.defaultBestBy, UseBy: c.config.defaultUseBy}
}

func newItemConfig(config *Config, options ...item.ConfigOption) *item.Config {
	itemConfig := &item.Config{BestBy: config.defaultBestBy, UseBy: config.defaultUseBy}
	for _, option := range options {
		option(itemConfig)
	}
	return itemConfig
}

func newQueryConfig(options ...item.QueryConfigOption) *item.QueryConfig {
	queryConfig := &item.QueryConfig{}
	for _, option := range options {
		option(queryConfig)
	}
	return queryConfig
}
<|endoftext|>


<commit_before>package hcsshim

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"

	"github.com/Microsoft/go-winio"
	"github.com/Sirupsen/logrus"
)

// ImportLayer will take the contents of the folder at importFolderPath and import
// that into a layer with the id layerId.  Note that in order to correctly populate
// the layer and interperet the transport format, all parent layers must already
// be present on the system at the paths provided in parentLayerPaths.
func ImportLayer(info DriverInfo, layerId string, importFolderPath string, parentLayerPaths []string) error {
	title := "hcsshim::ImportLayer "
	logrus.Debugf(title+"flavour %d layerId %s folder %s", info.Flavour, layerId, importFolderPath)

	// Generate layer descriptors
	layers, err := layerPathsToDescriptors(parentLayerPaths)
	if err != nil {
		return err
	}

	// Convert info to API calling convention
	infop, err := convertDriverInfo(info)
	if err != nil {
		logrus.Error(err)
		return err
	}

	err = importLayer(&infop, layerId, importFolderPath, layers)
	if err != nil {
		err = makeErrorf(err, title, "layerId=%s flavour=%d folder=%s", layerId, info.Flavour, importFolderPath)
		logrus.Error(err)
		return err
	}

	logrus.Debugf(title+"succeeded flavour=%d layerId=%s folder=%s", info.Flavour, layerId, importFolderPath)
	return nil
}

type LayerWriter interface {
	Add(name string, fileInfo *winio.FileBasicInfo) error
	Remove(name string) error
	Write(b []byte) (int, error)
	Close() error
}

// FilterLayerWriter provides an interface to write the contents of a layer to the file system.
type FilterLayerWriter struct {
	context uintptr
}

// Add adds a file or directory to the layer. The file's parent directory must have already been added.
//
// name contains the file's relative path. fileInfo contains file times and file attributes; the rest
// of the file metadata and the file data must be written as a Win32 backup stream to the Write() method.
// winio.BackupStreamWriter can be used to facilitate this.
func (w *FilterLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) error {
	if name[0] != '\\' {
		name = `\` + name
	}
	err := importLayerNext(w.context, name, fileInfo)
	if err != nil {
		return makeError(err, "ImportLayerNext", "")
	}
	return nil
}

// Remove removes a file from the layer. The file must have been present in the parent layer.
//
// name contains the file's relative path.
func (w *FilterLayerWriter) Remove(name string) error {
	if name[0] != '\\' {
		name = `\` + name
	}
	err := importLayerNext(w.context, name, nil)
	if err != nil {
		return makeError(err, "ImportLayerNext", "")
	}
	return nil
}

// Write writes more backup stream data to the current file.
func (w *FilterLayerWriter) Write(b []byte) (int, error) {
	err := importLayerWrite(w.context, b)
	if err != nil {
		err = makeError(err, "ImportLayerWrite", "")
		return 0, err
	}
	return len(b), err
}

// Close completes the layer write operation. The error must be checked to ensure that the
// operation was successful.
func (w *FilterLayerWriter) Close() (err error) {
	if w.context != 0 {
		err = importLayerEnd(w.context)
		if err != nil {
			err = makeError(err, "ImportLayerEnd", "")
		}
		w.context = 0
	}
	return
}

type legacyLayerWriterWrapper struct {
	*LegacyLayerWriter
	info             DriverInfo
	layerId          string
	path             string
	parentLayerPaths []string
}

func (r *legacyLayerWriterWrapper) Close() error {
	err := r.LegacyLayerWriter.Close()
	if err == nil {
		// Use the original path here because ImportLayer does not support long paths for the source in TP5.
		// But do use a long path for the destination to work around another bug with directories
		// with MAX_PATH - 12 < length < MAX_PATH.
		info := r.info
		fullPath, err := makeLongAbsPath(filepath.Join(info.HomeDir, r.layerId))
		if err == nil {
			info.HomeDir = ""
			err = ImportLayer(info, fullPath, r.path, r.parentLayerPaths)
		}
	}
	os.RemoveAll(r.root)
	return err
}

// NewLayerWriter returns a new layer writer for creating a layer on disk.
func NewLayerWriter(info DriverInfo, layerId string, parentLayerPaths []string) (LayerWriter, error) {
	if procImportLayerBegin.Find() != nil {
		// The new layer reader is not available on this Windows build. Fall back to the
		// legacy export code path.
		path, err := ioutil.TempDir("", "hcs")
		if err != nil {
			return nil, err
		}
		return &legacyLayerWriterWrapper{
			LegacyLayerWriter: NewLegacyLayerWriter(path),
			info:              info,
			layerId:           layerId,
			path:              path,
			parentLayerPaths:  parentLayerPaths,
		}, nil
	}
	layers, err := layerPathsToDescriptors(parentLayerPaths)
	if err != nil {
		return nil, err
	}

	infop, err := convertDriverInfo(info)
	if err != nil {
		return nil, err
	}

	w := &FilterLayerWriter{}
	err = importLayerBegin(&infop, layerId, layers, &w.context)
	if err != nil {
		return nil, makeError(err, "ImportLayerStart", "")
	}
	runtime.SetFinalizer(w, func(w *FilterLayerWriter) { w.Close() })
	return w, nil
}
<commit_msg>Don't swallow ImportLayer errors<commit_after>package hcsshim

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"

	"github.com/Microsoft/go-winio"
	"github.com/Sirupsen/logrus"
)

// ImportLayer will take the contents of the folder at importFolderPath and import
// that into a layer with the id layerId.  Note that in order to correctly populate
// the layer and interperet the transport format, all parent layers must already
// be present on the system at the paths provided in parentLayerPaths.
func ImportLayer(info DriverInfo, layerId string, importFolderPath string, parentLayerPaths []string) error {
	title := "hcsshim::ImportLayer "
	logrus.Debugf(title+"flavour %d layerId %s folder %s", info.Flavour, layerId, importFolderPath)

	// Generate layer descriptors
	layers, err := layerPathsToDescriptors(parentLayerPaths)
	if err != nil {
		return err
	}

	// Convert info to API calling convention
	infop, err := convertDriverInfo(info)
	if err != nil {
		logrus.Error(err)
		return err
	}

	err = importLayer(&infop, layerId, importFolderPath, layers)
	if err != nil {
		err = makeErrorf(err, title, "layerId=%s flavour=%d folder=%s", layerId, info.Flavour, importFolderPath)
		logrus.Error(err)
		return err
	}

	logrus.Debugf(title+"succeeded flavour=%d layerId=%s folder=%s", info.Flavour, layerId, importFolderPath)
	return nil
}

type LayerWriter interface {
	Add(name string, fileInfo *winio.FileBasicInfo) error
	Remove(name string) error
	Write(b []byte) (int, error)
	Close() error
}

// FilterLayerWriter provides an interface to write the contents of a layer to the file system.
type FilterLayerWriter struct {
	context uintptr
}

// Add adds a file or directory to the layer. The file's parent directory must have already been added.
//
// name contains the file's relative path. fileInfo contains file times and file attributes; the rest
// of the file metadata and the file data must be written as a Win32 backup stream to the Write() method.
// winio.BackupStreamWriter can be used to facilitate this.
func (w *FilterLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) error {
	if name[0] != '\\' {
		name = `\` + name
	}
	err := importLayerNext(w.context, name, fileInfo)
	if err != nil {
		return makeError(err, "ImportLayerNext", "")
	}
	return nil
}

// Remove removes a file from the layer. The file must have been present in the parent layer.
//
// name contains the file's relative path.
func (w *FilterLayerWriter) Remove(name string) error {
	if name[0] != '\\' {
		name = `\` + name
	}
	err := importLayerNext(w.context, name, nil)
	if err != nil {
		return makeError(err, "ImportLayerNext", "")
	}
	return nil
}

// Write writes more backup stream data to the current file.
func (w *FilterLayerWriter) Write(b []byte) (int, error) {
	err := importLayerWrite(w.context, b)
	if err != nil {
		err = makeError(err, "ImportLayerWrite", "")
		return 0, err
	}
	return len(b), err
}

// Close completes the layer write operation. The error must be checked to ensure that the
// operation was successful.
func (w *FilterLayerWriter) Close() (err error) {
	if w.context != 0 {
		err = importLayerEnd(w.context)
		if err != nil {
			err = makeError(err, "ImportLayerEnd", "")
		}
		w.context = 0
	}
	return
}

type legacyLayerWriterWrapper struct {
	*LegacyLayerWriter
	info             DriverInfo
	layerId          string
	path             string
	parentLayerPaths []string
}

func (r *legacyLayerWriterWrapper) Close() error {
	err := r.LegacyLayerWriter.Close()
	if err == nil {
		var fullPath string
		// Use the original path here because ImportLayer does not support long paths for the source in TP5.
		// But do use a long path for the destination to work around another bug with directories
		// with MAX_PATH - 12 < length < MAX_PATH.
		info := r.info
		fullPath, err = makeLongAbsPath(filepath.Join(info.HomeDir, r.layerId))
		if err == nil {
			info.HomeDir = ""
			err = ImportLayer(info, fullPath, r.path, r.parentLayerPaths)
		}
	}
	os.RemoveAll(r.root)
	return err
}

// NewLayerWriter returns a new layer writer for creating a layer on disk.
func NewLayerWriter(info DriverInfo, layerId string, parentLayerPaths []string) (LayerWriter, error) {
	if procImportLayerBegin.Find() != nil {
		// The new layer reader is not available on this Windows build. Fall back to the
		// legacy export code path.
		path, err := ioutil.TempDir("", "hcs")
		if err != nil {
			return nil, err
		}
		return &legacyLayerWriterWrapper{
			LegacyLayerWriter: NewLegacyLayerWriter(path),
			info:              info,
			layerId:           layerId,
			path:              path,
			parentLayerPaths:  parentLayerPaths,
		}, nil
	}
	layers, err := layerPathsToDescriptors(parentLayerPaths)
	if err != nil {
		return nil, err
	}

	infop, err := convertDriverInfo(info)
	if err != nil {
		return nil, err
	}

	w := &FilterLayerWriter{}
	err = importLayerBegin(&infop, layerId, layers, &w.context)
	if err != nil {
		return nil, makeError(err, "ImportLayerStart", "")
	}
	runtime.SetFinalizer(w, func(w *FilterLayerWriter) { w.Close() })
	return w, nil
}
<|endoftext|>


<commit_before>package log

import (
	"fmt"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/treetopllc/elastilog"
)

type ElasticClient struct {
	Basic
	client elastilog.Client
}

func NewElastic(uri string, tags ...string) *ElasticClient {
	c := &ElasticClient{
		client: elastilog.NewClient(uri, tags...),
	}
	c.Basic = NewBasic(c, "", 0)
	return c
}

func (el ElasticClient) Send(e *ElasticEntry) {
	if e != nil {
		e.entry.Attributes["level"] = string(e.level)
		e.entry.Log = string(e.str)
		el.client.Send(e.entry)
	}
}

func (el ElasticClient) Write(msg []byte) (int, error) {
	hostname, _ := os.Hostname()
	el.client.Send(elastilog.Entry{
		Log:        string(msg),
		Host:       hostname,
		Timestamp:  time.Now(),
		Attributes: elastilog.Attributes{"level": "std"},
	})
	return len(msg), nil
}

type ElasticEntry struct {
	level LogLevel
	str   StringLogger
	entry elastilog.Entry
}

func NewElasticEntry() *ElasticEntry {
	hostname, _ := os.Hostname()
	return &ElasticEntry{
		level: INFO,
		entry: elastilog.Entry{
			Host:       hostname,
			Timestamp:  time.Now(),
			Attributes: make(elastilog.Attributes),
		},
	}
}

func (ee *ElasticEntry) set(key string, value string) {
	ee.entry.Attributes[key] = value
}

func (ee *ElasticEntry) SetRequest(req *http.Request) {
	ee.entry.Host = req.URL.Host
	ee.set("request.method", req.Method)
	ee.set("request.path", req.URL.Path)
	ee.set("request.query", req.URL.RawQuery)
	ee.set("request.proto", req.Proto)
	if req.Body != nil {
		rb := requestBody(req)
		ee.set("request.body", rb+" ")
	}
	for k, h := range req.Header {
		ee.set("request.header."+k, strings.Join(h, ","))
	}
}
func (ee *ElasticEntry) SetUserID(id string) {
	ee.set("request.user_id", id)
}
func (ee *ElasticEntry) SetProductType(pt string) {
	ee.set("request.product_type", pt)
}

func (ee *ElasticEntry) SetResponse(status int, body interface{}) {
	if debug || status >= 300 {
		ee.set("response.body", responseBody(body))
	}
	ee.set("response.status", fmt.Sprintf("%v", status))
	ee.set("duration", fmt.Sprintf("%v", time.Since(ee.entry.Timestamp).Nanoseconds()/1000000)) //1 ms = 1000000ns
}

func (ee *ElasticEntry) Write(b []byte) (int, error) {
	ee.level.Set(INFO)
	ee.str.Log(string(b))
	return len(b), nil
}
func (ee *ElasticEntry) Log(msg ...interface{}) {
	ee.level.Set(INFO)
	ee.str.Log(msg...)
}
func (ee *ElasticEntry) Logf(msg string, args ...interface{}) {
	ee.level.Set(INFO)
	ee.str.Logf(msg, args...)
}
func (ee *ElasticEntry) Error(err error) {
	ee.level.Set(ERROR)
	ee.str.Error(err)
}
func (ee *ElasticEntry) Errorf(msg string, args ...interface{}) {
	ee.level.Set(ERROR)
	ee.str.Errorf(msg, args...)
}
func (ee *ElasticEntry) Panic(msg interface{}) {
	ee.level.Set(PANIC)
	ee.str.Panic(msg)
}
func (ee *ElasticEntry) Panicf(msg string, args ...interface{}) {
	ee.level.Set(PANIC)
	ee.str.Panicf(msg, args...)
}
<commit_msg>Include request IDs in elastic logs<commit_after>package log

import (
	"fmt"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/treetopllc/elastilog"
)

type ElasticClient struct {
	Basic
	client elastilog.Client
}

func NewElastic(uri string, tags ...string) *ElasticClient {
	c := &ElasticClient{
		client: elastilog.NewClient(uri, tags...),
	}
	c.Basic = NewBasic(c, "", 0)
	return c
}

func (el ElasticClient) Send(e *ElasticEntry) {
	if e != nil {
		e.entry.Attributes["level"] = string(e.level)
		e.entry.Log = string(e.str)
		el.client.Send(e.entry)
	}
}

func (el ElasticClient) Write(msg []byte) (int, error) {
	hostname, _ := os.Hostname()
	el.client.Send(elastilog.Entry{
		Log:        string(msg),
		Host:       hostname,
		Timestamp:  time.Now(),
		Attributes: elastilog.Attributes{"level": "std"},
	})
	return len(msg), nil
}

type ElasticEntry struct {
	level LogLevel
	str   StringLogger
	entry elastilog.Entry
}

func NewElasticEntry() *ElasticEntry {
	hostname, _ := os.Hostname()
	return &ElasticEntry{
		level: INFO,
		entry: elastilog.Entry{
			Host:       hostname,
			Timestamp:  time.Now(),
			Attributes: make(elastilog.Attributes),
		},
	}
}

func (ee *ElasticEntry) set(key string, value string) {
	ee.entry.Attributes[key] = value
}

func (ee *ElasticEntry) SetRequest(req *http.Request) {
	ee.entry.Host = req.URL.Host
	ee.set("request.method", req.Method)
	ee.set("request.path", req.URL.Path)
	ee.set("request.query", req.URL.RawQuery)
	ee.set("request.proto", req.Proto)
	if req.Body != nil {
		rb := requestBody(req)
		ee.set("request.body", rb+" ")
	}
	for k, h := range req.Header {
		ee.set("request.header."+k, strings.Join(h, ","))
	}
}
func (ee *ElasticEntry) SetUserID(id string) {
	ee.set("request.user_id", id)
}
func (ee *ElasticEntry) SetProductType(pt string) {
	ee.set("request.product_type", pt)
}
func (ee *ElasticEntry) SetRequestID(id string) {
	ee.set("request.id", id)
}

func (ee *ElasticEntry) SetResponse(status int, body interface{}) {
	if debug || status >= 300 {
		ee.set("response.body", responseBody(body))
	}
	ee.set("response.status", fmt.Sprintf("%v", status))
	ee.set("duration", fmt.Sprintf("%v", time.Since(ee.entry.Timestamp).Nanoseconds()/1000000)) //1 ms = 1000000ns
}

func (ee *ElasticEntry) Write(b []byte) (int, error) {
	ee.level.Set(INFO)
	ee.str.Log(string(b))
	return len(b), nil
}
func (ee *ElasticEntry) Log(msg ...interface{}) {
	ee.level.Set(INFO)
	ee.str.Log(msg...)
}
func (ee *ElasticEntry) Logf(msg string, args ...interface{}) {
	ee.level.Set(INFO)
	ee.str.Logf(msg, args...)
}
func (ee *ElasticEntry) Error(err error) {
	ee.level.Set(ERROR)
	ee.str.Error(err)
}
func (ee *ElasticEntry) Errorf(msg string, args ...interface{}) {
	ee.level.Set(ERROR)
	ee.str.Errorf(msg, args...)
}
func (ee *ElasticEntry) Panic(msg interface{}) {
	ee.level.Set(PANIC)
	ee.str.Panic(msg)
}
func (ee *ElasticEntry) Panicf(msg string, args ...interface{}) {
	ee.level.Set(PANIC)
	ee.str.Panicf(msg, args...)
}
<|endoftext|>


<commit_before><commit_msg>converted needMoreError to static error. This may improve server performance in production if clients send request headers in chunks<commit_after><|endoftext|>


<commit_before>package client

import (
	"fmt"

	"github.com/kentik/libkflow"
	"github.com/kentik/libkflow/api"
)

type FlowClient struct {
	Sender          *libkflow.Sender
	SetSrcHostTags  map[string]bool
	SetDestHostTags map[string]bool
	interfaces      map[string]api.InterfaceUpdate
	idsByAlias      map[string]uint32
	doneInit        bool
	nextInterface   uint32
}

func NewFlowClient(client *libkflow.Sender) *FlowClient {
	return &FlowClient{
		Sender:          client,
		SetSrcHostTags:  map[string]bool{},
		SetDestHostTags: map[string]bool{},
		idsByAlias: map[string]uint32{
			"": 1,
		},
		interfaces: map[string]api.InterfaceUpdate{
			"eth0": api.InterfaceUpdate{ // Pre-populate this with eth0 for now.
				Index:   1,
				Desc:    "eth0",
				Alias:   "",
				Address: "127.0.0.1",
			},
		},
		nextInterface: 2,
	}
}

func (c *FlowClient) ResetTags() {
	c.SetSrcHostTags = map[string]bool{}
	c.SetDestHostTags = map[string]bool{}
}

func (c *FlowClient) GetInterfaceID(host string) uint32 {
	return c.idsByAlias[host]
}

func (c *FlowClient) UpdateInterfaces(isFromInterfaceUpdate bool) error {

	// Only run from not interfaces once
	if c.doneInit && !isFromInterfaceUpdate {
		return nil
	}
	c.doneInit = true

	client := c.Sender.GetClient()
	if client != nil {
		err := client.UpdateInterfacesDirectly(c.Sender.Device, c.interfaces)
		if err != nil {
			return err
		}
	}
	return nil
}

func (c *FlowClient) AddInterface(intf *api.InterfaceUpdate) {
	intf.Index = uint64(c.nextInterface)
	c.idsByAlias[intf.Alias] = c.nextInterface
	intf.Desc = fmt.Sprintf("kentik.%d", c.nextInterface)
	c.nextInterface++

	c.interfaces[intf.Desc] = *intf
}
<commit_msg>Mapping internal and eternal traffic now using eth0 and int0<commit_after>package client

import (
	"fmt"

	"github.com/kentik/libkflow"
	"github.com/kentik/libkflow/api"
)

type FlowClient struct {
	Sender          *libkflow.Sender
	SetSrcHostTags  map[string]bool
	SetDestHostTags map[string]bool
	interfaces      map[string]api.InterfaceUpdate
	idsByAlias      map[string]uint32
	doneInit        bool
	nextInterface   uint32
}

func NewFlowClient(client *libkflow.Sender) *FlowClient {
	return &FlowClient{
		Sender:          client,
		SetSrcHostTags:  map[string]bool{},
		SetDestHostTags: map[string]bool{},
		idsByAlias: map[string]uint32{
			"": 1,
		},
		interfaces: map[string]api.InterfaceUpdate{
			"eth0": api.InterfaceUpdate{ // Pre-populate this with eth0 for now for external traffic
				Index:   1,
				Desc:    "eth0",
				Alias:   "",
				Address: "127.0.0.1",
			},
			"int0": api.InterfaceUpdate{ // Pre-populate this with int1 for internal traffic.
				Index:   2,
				Desc:    "int0",
				Alias:   "",
				Address: "127.0.0.2",
			},
		},
		nextInterface: 3,
	}
}

func (c *FlowClient) ResetTags() {
	c.SetSrcHostTags = map[string]bool{}
	c.SetDestHostTags = map[string]bool{}
}

func (c *FlowClient) GetInterfaceID(host string) uint32 {
	if id, ok := c.idsByAlias[host]; ok {
		return id
	} else {
		return c.idsByAlias["int0"] // Known vm, but not on this host, so we send out the int0 interface.
	}
}

func (c *FlowClient) UpdateInterfaces(isFromInterfaceUpdate bool) error {

	// Only run from not interfaces once
	if c.doneInit && !isFromInterfaceUpdate {
		return nil
	}
	c.doneInit = true

	client := c.Sender.GetClient()
	if client != nil {
		err := client.UpdateInterfacesDirectly(c.Sender.Device, c.interfaces)
		if err != nil {
			return err
		}
	}
	return nil
}

func (c *FlowClient) AddInterface(intf *api.InterfaceUpdate) {
	intf.Index = uint64(c.nextInterface)
	c.idsByAlias[intf.Alias] = c.nextInterface
	intf.Desc = fmt.Sprintf("kentik.%d", c.nextInterface)
	c.nextInterface++

	c.interfaces[intf.Desc] = *intf
}
<|endoftext|>


<commit_before>package metrics

import (
	"github.com/magneticio/vamp-router/haproxy"
	gologger "github.com/op/go-logging"
	"strconv"
	"strings"
	"sync/atomic"
	"time"
)

type Streamer struct {

	// simple counter to give heartbeats in the log how many messages where parsed during a time period
	Counter       int64
	wantedMetrics []string
	haRuntime     *haproxy.Runtime
	pollFrequency int
	Clients       map[chan Metric]bool
	Log           *gologger.Logger
}

// Adds a client to which messages can be multiplexed.
func (s *Streamer) AddClient(c chan Metric) {
	s.Clients[c] = true
}

// Just sets the metrics we want for now...
func (s *Streamer) Init(haRuntime *haproxy.Runtime, frequency int, log *gologger.Logger) {
	s.Log = log
	s.wantedMetrics = []string{"scur", "qcur", "qmax", "smax", "slim", "ereq", "econ", "lastsess", "qtime", "ctime", "rtime", "ttime", "req_rate", "req_rate_max", "req_tot", "rate", "rate_lim", "rate_max", "hrsp_1xx", "hrsp_2xx", "hrsp_3xx", "hrsp_4xx", "hrsp_5xx"}
	s.haRuntime = haRuntime
	s.pollFrequency = frequency
	s.Clients = make(map[chan Metric]bool)
}

/*
  Generates an outgoing stream of discrete Metric struct values.
  This stream can then be consumed by other streams like Kafka or SSE.
*/
func (s *Streamer) Start() error {

	s.Heartbeat()

	// create a channel to send the stats to the parser
	statsChannel := make(chan map[string]map[string]string)

	// start up the parser in a separate routine
	go ParseMetrics(statsChannel, s.Clients, s.wantedMetrics, &s.Counter)

	for {
		// start pumping the stats into the channel
		stats, _ := s.haRuntime.GetStats("all")
		statsChannel <- stats
		time.Sleep(time.Duration(s.pollFrequency) * time.Millisecond)
	}
}

/*
	Parses a []Stats and injects it into each Metric channel in a map of channels
*/

func ParseMetrics(statsChannel chan map[string]map[string]string, c map[chan Metric]bool, wantedMetrics []string, counter *int64) {

	wantedFrontendMetric := make(map[string]bool)
	wantedFrontendMetric["ereq"] = true
	wantedFrontendMetric["rate_lim"] = true
	wantedFrontendMetric["req_rate_max"] = true

	for {

		select {

		case stats := <-statsChannel:

			localTime := time.Now().Format(time.RFC3339)

			// for each proxy in the stats dump, pick out the wanted metrics.
			for _, proxy := range stats {

				// loop over all wanted metrics for the current proxy
				for _, metric := range wantedMetrics {

					// discard all empty metrics
					if proxy[metric] != "" {

						value := proxy[metric]
						svname := proxy["svname"]
						tags := []string{}
						pxnames := strings.Split(proxy["pxname"], ".")

						// allow only some FRONTEND metrics and all non-FRONTEND metrics
						if (svname == "FRONTEND" && wantedFrontendMetric[metric]) || svname != "FRONTEND" {

							// Compile tags
							// we tag the metrics according to the followin scheme

							//- if pxname has no "." separator, and svname is [BACKEND|FRONTEND] it is the top route
							if len(pxnames) == 1 && (svname == "BACKEND" || svname == "FRONTEND") {
								tags = append(tags, "route:"+proxy["pxname"])
							} else {

								//-if pxname has no "."  separator, and svname is not [BACKEND|FRONTEND] it is an "in between"
								// server that routes to the actual service via a socket.
								if len(pxnames) == 1 && (svname != "BACKEND" || svname != "FRONTEND") {
									sockName := strings.Split(svname, ".")
									tags = append(tags, "route:"+proxy["pxname"], "socket_server:"+sockName[1])
								} else {

									//- if pxname has a separator, and svname is [BACKEND|FRONTEND] it is a service
									if len(pxnames) > 1 && (svname == "BACKEND" || svname == "FRONTEND") {
										tags = append(tags, "route:"+pxnames[0], "service:"+pxnames[1])
									} else {

										//- if svname is not [BACKEND|FRONTEND] its a SERVER in a SERVICE and we prepend it with "server:"
										if len(pxnames) > 1 && (svname != "BACKEND" && svname != "FRONTEND") {
											tags = append(tags, "route:"+pxnames[0], "service:"+pxnames[1], "server:"+svname)
										}
									}
								}
							}

							tags = append(tags, "metric:"+metric)

							metricValue, _ := strconv.Atoi(value)
							metric := Metric{tags, metricValue, localTime}
							atomic.AddInt64(counter, 1)

							for s, _ := range c {
								s <- metric
							}
						}
					}
				}
			}
		}
	}
}

/*
  Logs a message every ticker interval giving an update on how many messages were parsed
*/
func (s *Streamer) Heartbeat() error {

	ticker := time.NewTicker(60 * time.Second)

	go func() {
		for {
			select {
			case <-ticker.C:
				s.Log.Notice("Metrics parsed in last minute: %d \n", s.Counter)
				atomic.StoreInt64(&s.Counter, 0)
			}
		}
	}()
	return nil
}
<commit_msg>made tags plural as per convention<commit_after>package metrics

import (
	"github.com/magneticio/vamp-router/haproxy"
	gologger "github.com/op/go-logging"
	"strconv"
	"strings"
	"sync/atomic"
	"time"
)

type Streamer struct {

	// simple counter to give heartbeats in the log how many messages where parsed during a time period
	Counter       int64
	wantedMetrics []string
	haRuntime     *haproxy.Runtime
	pollFrequency int
	Clients       map[chan Metric]bool
	Log           *gologger.Logger
}

// Adds a client to which messages can be multiplexed.
func (s *Streamer) AddClient(c chan Metric) {
	s.Clients[c] = true
}

// Just sets the metrics we want for now...
func (s *Streamer) Init(haRuntime *haproxy.Runtime, frequency int, log *gologger.Logger) {
	s.Log = log
	s.wantedMetrics = []string{"scur", "qcur", "qmax", "smax", "slim", "ereq", "econ", "lastsess", "qtime", "ctime", "rtime", "ttime", "req_rate", "req_rate_max", "req_tot", "rate", "rate_lim", "rate_max", "hrsp_1xx", "hrsp_2xx", "hrsp_3xx", "hrsp_4xx", "hrsp_5xx"}
	s.haRuntime = haRuntime
	s.pollFrequency = frequency
	s.Clients = make(map[chan Metric]bool)
}

/*
  Generates an outgoing stream of discrete Metric struct values.
  This stream can then be consumed by other streams like Kafka or SSE.
*/
func (s *Streamer) Start() error {

	s.Heartbeat()

	// create a channel to send the stats to the parser
	statsChannel := make(chan map[string]map[string]string)

	// start up the parser in a separate routine
	go ParseMetrics(statsChannel, s.Clients, s.wantedMetrics, &s.Counter)

	for {
		// start pumping the stats into the channel
		stats, _ := s.haRuntime.GetStats("all")
		statsChannel <- stats
		time.Sleep(time.Duration(s.pollFrequency) * time.Millisecond)
	}
}

/*
	Parses a []Stats and injects it into each Metric channel in a map of channels
*/

func ParseMetrics(statsChannel chan map[string]map[string]string, c map[chan Metric]bool, wantedMetrics []string, counter *int64) {

	wantedFrontendMetric := make(map[string]bool)
	wantedFrontendMetric["ereq"] = true
	wantedFrontendMetric["rate_lim"] = true
	wantedFrontendMetric["req_rate_max"] = true

	for {

		select {

		case stats := <-statsChannel:

			localTime := time.Now().Format(time.RFC3339)

			// for each proxy in the stats dump, pick out the wanted metrics.
			for _, proxy := range stats {

				// loop over all wanted metrics for the current proxy
				for _, metric := range wantedMetrics {

					// discard all empty metrics
					if proxy[metric] != "" {

						value := proxy[metric]
						svname := proxy["svname"]
						tags := []string{}
						pxnames := strings.Split(proxy["pxname"], ".")

						// allow only some FRONTEND metrics and all non-FRONTEND metrics
						if (svname == "FRONTEND" && wantedFrontendMetric[metric]) || svname != "FRONTEND" {

							// Compile tags
							// we tag the metrics according to the followin scheme

							//- if pxname has no "." separator, and svname is [BACKEND|FRONTEND] it is the top route
							if len(pxnames) == 1 && (svname == "BACKEND" || svname == "FRONTEND") {
								tags = append(tags, "routes:"+proxy["pxname"])
							} else {

								//-if pxname has no "."  separator, and svname is not [BACKEND|FRONTEND] it is an "in between"
								// server that routes to the actual service via a socket.
								if len(pxnames) == 1 && (svname != "BACKEND" || svname != "FRONTEND") {
									sockName := strings.Split(svname, ".")
									tags = append(tags, "routes:"+proxy["pxname"], "socket_servers:"+sockName[1])
								} else {

									//- if pxname has a separator, and svname is [BACKEND|FRONTEND] it is a service
									if len(pxnames) > 1 && (svname == "BACKEND" || svname == "FRONTEND") {
										tags = append(tags, "routes:"+pxnames[0], "services:"+pxnames[1])
									} else {

										//- if svname is not [BACKEND|FRONTEND] its a SERVER in a SERVICE and we prepend it with "server:"
										if len(pxnames) > 1 && (svname != "BACKEND" && svname != "FRONTEND") {
											tags = append(tags, "routes:"+pxnames[0], "services:"+pxnames[1], "servers:"+svname)
										}
									}
								}
							}

							tags = append(tags, "metrics:"+metric)

							metricValue, _ := strconv.Atoi(value)
							metric := Metric{tags, metricValue, localTime}
							atomic.AddInt64(counter, 1)

							for s, _ := range c {
								s <- metric
							}
						}
					}
				}
			}
		}
	}
}

/*
  Logs a message every ticker interval giving an update on how many messages were parsed
*/
func (s *Streamer) Heartbeat() error {

	ticker := time.NewTicker(60 * time.Second)

	go func() {
		for {
			select {
			case <-ticker.C:
				s.Log.Notice("Metrics parsed in last minute: %d \n", s.Counter)
				atomic.StoreInt64(&s.Counter, 0)
			}
		}
	}()
	return nil
}
<|endoftext|>


<commit_before>// Package registry contains client primitives to interact with a remote Docker registry.
package registry

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/distribution/registry/api/errcode"
	"github.com/docker/distribution/registry/api/v2"
	"github.com/docker/distribution/registry/client"
	"github.com/docker/distribution/registry/client/transport"
	"github.com/docker/docker/dockerversion"
	"github.com/docker/docker/pkg/parsers/kernel"
	"github.com/docker/docker/pkg/tlsconfig"
	"github.com/docker/docker/pkg/useragent"
)

var (
	// ErrAlreadyExists is an error returned if an image being pushed
	// already exists on the remote side
	ErrAlreadyExists = errors.New("Image already exists")
	errLoginRequired = errors.New("Authentication is required.")
)

// dockerUserAgent is the User-Agent the Docker client uses to identify itself.
// It is populated on init(), comprising version information of different components.
var dockerUserAgent string

func init() {
	httpVersion := make([]useragent.VersionInfo, 0, 6)
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "docker", Version: dockerversion.Version})
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "go", Version: runtime.Version()})
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "git-commit", Version: dockerversion.GitCommit})
	if kernelVersion, err := kernel.GetKernelVersion(); err == nil {
		httpVersion = append(httpVersion, useragent.VersionInfo{Name: "kernel", Version: kernelVersion.String()})
	}
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "os", Version: runtime.GOOS})
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "arch", Version: runtime.GOARCH})

	dockerUserAgent = useragent.AppendVersions("", httpVersion...)

	if runtime.GOOS != "linux" {
		V2Only = true
	}
}

func newTLSConfig(hostname string, isSecure bool) (*tls.Config, error) {
	// PreferredServerCipherSuites should have no effect
	tlsConfig := tlsconfig.ServerDefault

	tlsConfig.InsecureSkipVerify = !isSecure

	if isSecure {
		hostDir := filepath.Join(CertsDir, cleanPath(hostname))
		logrus.Debugf("hostDir: %s", hostDir)
		if err := ReadCertsDirectory(&tlsConfig, hostDir); err != nil {
			return nil, err
		}
	}

	return &tlsConfig, nil
}

func hasFile(files []os.FileInfo, name string) bool {
	for _, f := range files {
		if f.Name() == name {
			return true
		}
	}
	return false
}

// ReadCertsDirectory reads the directory for TLS certificates
// including roots and certificate pairs and updates the
// provided TLS configuration.
func ReadCertsDirectory(tlsConfig *tls.Config, directory string) error {
	fs, err := ioutil.ReadDir(directory)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	for _, f := range fs {
		if strings.HasSuffix(f.Name(), ".crt") {
			if tlsConfig.RootCAs == nil {
				// TODO(dmcgowan): Copy system pool
				tlsConfig.RootCAs = x509.NewCertPool()
			}
			logrus.Debugf("crt: %s", filepath.Join(directory, f.Name()))
			data, err := ioutil.ReadFile(filepath.Join(directory, f.Name()))
			if err != nil {
				return err
			}
			tlsConfig.RootCAs.AppendCertsFromPEM(data)
		}
		if strings.HasSuffix(f.Name(), ".cert") {
			certName := f.Name()
			keyName := certName[:len(certName)-5] + ".key"
			logrus.Debugf("cert: %s", filepath.Join(directory, f.Name()))
			if !hasFile(fs, keyName) {
				return fmt.Errorf("Missing key %s for certificate %s", keyName, certName)
			}
			cert, err := tls.LoadX509KeyPair(filepath.Join(directory, certName), filepath.Join(directory, keyName))
			if err != nil {
				return err
			}
			tlsConfig.Certificates = append(tlsConfig.Certificates, cert)
		}
		if strings.HasSuffix(f.Name(), ".key") {
			keyName := f.Name()
			certName := keyName[:len(keyName)-4] + ".cert"
			logrus.Debugf("key: %s", filepath.Join(directory, f.Name()))
			if !hasFile(fs, certName) {
				return fmt.Errorf("Missing certificate %s for key %s", certName, keyName)
			}
		}
	}

	return nil
}

// DockerHeaders returns request modifiers that ensure requests have
// the User-Agent header set to dockerUserAgent and that metaHeaders
// are added.
func DockerHeaders(metaHeaders http.Header) []transport.RequestModifier {
	modifiers := []transport.RequestModifier{
		transport.NewHeaderRequestModifier(http.Header{"User-Agent": []string{dockerUserAgent}}),
	}
	if metaHeaders != nil {
		modifiers = append(modifiers, transport.NewHeaderRequestModifier(metaHeaders))
	}
	return modifiers
}

// HTTPClient returns a HTTP client structure which uses the given transport
// and contains the necessary headers for redirected requests
func HTTPClient(transport http.RoundTripper) *http.Client {
	return &http.Client{
		Transport:     transport,
		CheckRedirect: addRequiredHeadersToRedirectedRequests,
	}
}

func trustedLocation(req *http.Request) bool {
	var (
		trusteds = []string{"docker.com", "docker.io"}
		hostname = strings.SplitN(req.Host, ":", 2)[0]
	)
	if req.URL.Scheme != "https" {
		return false
	}

	for _, trusted := range trusteds {
		if hostname == trusted || strings.HasSuffix(hostname, "."+trusted) {
			return true
		}
	}
	return false
}

// addRequiredHeadersToRedirectedRequests adds the necessary redirection headers
// for redirected requests
func addRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Request) error {
	if via != nil && via[0] != nil {
		if trustedLocation(req) && trustedLocation(via[0]) {
			req.Header = via[0].Header
			return nil
		}
		for k, v := range via[0].Header {
			if k != "Authorization" {
				for _, vv := range v {
					req.Header.Add(k, vv)
				}
			}
		}
	}
	return nil
}

func shouldV2Fallback(err errcode.Error) bool {
	logrus.Debugf("v2 error: %T %v", err, err)
	switch err.Code {
	case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown:
		return true
	}
	return false
}

// ErrNoSupport is an error type used for errors indicating that an operation
// is not supported. It encapsulates a more specific error.
type ErrNoSupport struct{ Err error }

func (e ErrNoSupport) Error() string {
	if e.Err == nil {
		return "not supported"
	}
	return e.Err.Error()
}

// ContinueOnError returns true if we should fallback to the next endpoint
// as a result of this error.
func ContinueOnError(err error) bool {
	switch v := err.(type) {
	case errcode.Errors:
		return ContinueOnError(v[0])
	case ErrNoSupport:
		return ContinueOnError(v.Err)
	case errcode.Error:
		return shouldV2Fallback(v)
	case *client.UnexpectedHTTPResponseError:
		return true
	case error:
		if val := strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error())); val {
			return false
		}
	}
	// let's be nice and fallback if the error is a completely
	// unexpected one.
	// If new errors have to be handled in some way, please
	// add them to the switch above.
	return true
}

// NewTransport returns a new HTTP transport. If tlsConfig is nil, it uses the
// default TLS configuration.
func NewTransport(tlsConfig *tls.Config) *http.Transport {
	if tlsConfig == nil {
		var cfg = tlsconfig.ServerDefault
		tlsConfig = &cfg
	}
	return &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig:     tlsConfig,
		// TODO(dmcgowan): Call close idle connections when complete and use keep alive
		DisableKeepAlives: true,
	}
}
<commit_msg>registry/registry.go: simplify logical expression<commit_after>// Package registry contains client primitives to interact with a remote Docker registry.
package registry

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/distribution/registry/api/errcode"
	"github.com/docker/distribution/registry/api/v2"
	"github.com/docker/distribution/registry/client"
	"github.com/docker/distribution/registry/client/transport"
	"github.com/docker/docker/dockerversion"
	"github.com/docker/docker/pkg/parsers/kernel"
	"github.com/docker/docker/pkg/tlsconfig"
	"github.com/docker/docker/pkg/useragent"
)

var (
	// ErrAlreadyExists is an error returned if an image being pushed
	// already exists on the remote side
	ErrAlreadyExists = errors.New("Image already exists")
	errLoginRequired = errors.New("Authentication is required.")
)

// dockerUserAgent is the User-Agent the Docker client uses to identify itself.
// It is populated on init(), comprising version information of different components.
var dockerUserAgent string

func init() {
	httpVersion := make([]useragent.VersionInfo, 0, 6)
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "docker", Version: dockerversion.Version})
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "go", Version: runtime.Version()})
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "git-commit", Version: dockerversion.GitCommit})
	if kernelVersion, err := kernel.GetKernelVersion(); err == nil {
		httpVersion = append(httpVersion, useragent.VersionInfo{Name: "kernel", Version: kernelVersion.String()})
	}
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "os", Version: runtime.GOOS})
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "arch", Version: runtime.GOARCH})

	dockerUserAgent = useragent.AppendVersions("", httpVersion...)

	if runtime.GOOS != "linux" {
		V2Only = true
	}
}

func newTLSConfig(hostname string, isSecure bool) (*tls.Config, error) {
	// PreferredServerCipherSuites should have no effect
	tlsConfig := tlsconfig.ServerDefault

	tlsConfig.InsecureSkipVerify = !isSecure

	if isSecure {
		hostDir := filepath.Join(CertsDir, cleanPath(hostname))
		logrus.Debugf("hostDir: %s", hostDir)
		if err := ReadCertsDirectory(&tlsConfig, hostDir); err != nil {
			return nil, err
		}
	}

	return &tlsConfig, nil
}

func hasFile(files []os.FileInfo, name string) bool {
	for _, f := range files {
		if f.Name() == name {
			return true
		}
	}
	return false
}

// ReadCertsDirectory reads the directory for TLS certificates
// including roots and certificate pairs and updates the
// provided TLS configuration.
func ReadCertsDirectory(tlsConfig *tls.Config, directory string) error {
	fs, err := ioutil.ReadDir(directory)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	for _, f := range fs {
		if strings.HasSuffix(f.Name(), ".crt") {
			if tlsConfig.RootCAs == nil {
				// TODO(dmcgowan): Copy system pool
				tlsConfig.RootCAs = x509.NewCertPool()
			}
			logrus.Debugf("crt: %s", filepath.Join(directory, f.Name()))
			data, err := ioutil.ReadFile(filepath.Join(directory, f.Name()))
			if err != nil {
				return err
			}
			tlsConfig.RootCAs.AppendCertsFromPEM(data)
		}
		if strings.HasSuffix(f.Name(), ".cert") {
			certName := f.Name()
			keyName := certName[:len(certName)-5] + ".key"
			logrus.Debugf("cert: %s", filepath.Join(directory, f.Name()))
			if !hasFile(fs, keyName) {
				return fmt.Errorf("Missing key %s for certificate %s", keyName, certName)
			}
			cert, err := tls.LoadX509KeyPair(filepath.Join(directory, certName), filepath.Join(directory, keyName))
			if err != nil {
				return err
			}
			tlsConfig.Certificates = append(tlsConfig.Certificates, cert)
		}
		if strings.HasSuffix(f.Name(), ".key") {
			keyName := f.Name()
			certName := keyName[:len(keyName)-4] + ".cert"
			logrus.Debugf("key: %s", filepath.Join(directory, f.Name()))
			if !hasFile(fs, certName) {
				return fmt.Errorf("Missing certificate %s for key %s", certName, keyName)
			}
		}
	}

	return nil
}

// DockerHeaders returns request modifiers that ensure requests have
// the User-Agent header set to dockerUserAgent and that metaHeaders
// are added.
func DockerHeaders(metaHeaders http.Header) []transport.RequestModifier {
	modifiers := []transport.RequestModifier{
		transport.NewHeaderRequestModifier(http.Header{"User-Agent": []string{dockerUserAgent}}),
	}
	if metaHeaders != nil {
		modifiers = append(modifiers, transport.NewHeaderRequestModifier(metaHeaders))
	}
	return modifiers
}

// HTTPClient returns a HTTP client structure which uses the given transport
// and contains the necessary headers for redirected requests
func HTTPClient(transport http.RoundTripper) *http.Client {
	return &http.Client{
		Transport:     transport,
		CheckRedirect: addRequiredHeadersToRedirectedRequests,
	}
}

func trustedLocation(req *http.Request) bool {
	var (
		trusteds = []string{"docker.com", "docker.io"}
		hostname = strings.SplitN(req.Host, ":", 2)[0]
	)
	if req.URL.Scheme != "https" {
		return false
	}

	for _, trusted := range trusteds {
		if hostname == trusted || strings.HasSuffix(hostname, "."+trusted) {
			return true
		}
	}
	return false
}

// addRequiredHeadersToRedirectedRequests adds the necessary redirection headers
// for redirected requests
func addRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Request) error {
	if via != nil && via[0] != nil {
		if trustedLocation(req) && trustedLocation(via[0]) {
			req.Header = via[0].Header
			return nil
		}
		for k, v := range via[0].Header {
			if k != "Authorization" {
				for _, vv := range v {
					req.Header.Add(k, vv)
				}
			}
		}
	}
	return nil
}

func shouldV2Fallback(err errcode.Error) bool {
	logrus.Debugf("v2 error: %T %v", err, err)
	switch err.Code {
	case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown:
		return true
	}
	return false
}

// ErrNoSupport is an error type used for errors indicating that an operation
// is not supported. It encapsulates a more specific error.
type ErrNoSupport struct{ Err error }

func (e ErrNoSupport) Error() string {
	if e.Err == nil {
		return "not supported"
	}
	return e.Err.Error()
}

// ContinueOnError returns true if we should fallback to the next endpoint
// as a result of this error.
func ContinueOnError(err error) bool {
	switch v := err.(type) {
	case errcode.Errors:
		return ContinueOnError(v[0])
	case ErrNoSupport:
		return ContinueOnError(v.Err)
	case errcode.Error:
		return shouldV2Fallback(v)
	case *client.UnexpectedHTTPResponseError:
		return true
	case error:
		return !strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error()))
	}
	// let's be nice and fallback if the error is a completely
	// unexpected one.
	// If new errors have to be handled in some way, please
	// add them to the switch above.
	return true
}

// NewTransport returns a new HTTP transport. If tlsConfig is nil, it uses the
// default TLS configuration.
func NewTransport(tlsConfig *tls.Config) *http.Transport {
	if tlsConfig == nil {
		var cfg = tlsconfig.ServerDefault
		tlsConfig = &cfg
	}
	return &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig:     tlsConfig,
		// TODO(dmcgowan): Call close idle connections when complete and use keep alive
		DisableKeepAlives: true,
	}
}
<|endoftext|>


<commit_before>package middleware

import (
	"github.com/afex/hystrix-go/hystrix"
	"github.com/gin-gonic/gin"
	"strings"

	"github.com/eirka/eirka-libs/redis"
)

var (
	RedisKeyIndex = make(map[string]RedisKey)
	RedisKeys     = []RedisKey{
		{base: "index", fieldcount: 1, hash: true, expire: false},
		{base: "thread", fieldcount: 2, hash: true, expire: false},
		{base: "tag", fieldcount: 2, hash: true, expire: true},
		{base: "image", fieldcount: 1, hash: true, expire: false},
		{base: "post", fieldcount: 2, hash: true, expire: false},
		{base: "tags", fieldcount: 1, hash: true, expire: false},
		{base: "directory", fieldcount: 1, hash: true, expire: false},
		{base: "new", fieldcount: 1, hash: false, expire: true},
		{base: "popular", fieldcount: 1, hash: false, expire: true},
		{base: "favorited", fieldcount: 1, hash: false, expire: true},
		{base: "tagtypes", fieldcount: 0, hash: false, expire: false},
		{base: "imageboards", fieldcount: 0, hash: false, expire: true},
	}
)

func init() {
	// key index map
	for _, key := range RedisKeys {
		RedisKeyIndex[key.base] = key
	}
}

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {
		var result []byte
		var err error

		// bool for analytics middleware
		c.Set("cached", false)

		// break cache if there is a query
		if c.Request.URL.RawQuery != "" {
			c.Next()
			return
		}

		// redis circuitbreaker
		err = hystrix.Do("cache", func() (err error) {

			// Trim leading / from path and split
			params := strings.Split(strings.Trim(c.Request.URL.Path, "/"), "/")

			// get the keyname
			key, ok := RedisKeyIndex[params[0]]
			if !ok {
				c.Next()
				return
			}

			// set the key minus the base
			key.SetKey(params[1:]...)

			result, err = key.Get()
			if err == redis.ErrCacheMiss {
				// go to the controller
				c.Next()

				// Check if there was an error from the controller
				_, controllerError := c.Get("controllerError")
				if controllerError {
					c.Abort()
					return
				}

				err = key.Set(c.MustGet("data").([]byte))
				if err != nil {
					c.Error(err)
					c.Abort()
					return
				}

			} else if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

			return

		}, func(err error) error {
			c.Next()
			return
		})

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()
		return
	}

}

type RedisKey struct {
	base       string
	fieldcount int
	hash       bool
	expire     bool
	key        string
	hashid     string
}

func (r *RedisKey) SetKey(ids ...string) {

	if r.fieldcount == 0 {
		r.key = r.base
		return
	}

	// create our key
	r.key = strings.Join([]string{r.base, strings.Join(ids[:r.fieldcount], ":")}, ":")

	// get our hash id
	if r.hash {
		r.hashid = strings.Join(ids[r.fieldcount:], "")
	}

	return
}

func (r *RedisKey) Get() (result []byte, err error) {

	if r.hash {
		return redis.RedisCache.HGet(r.key, r.hashid)
	} else {
		return redis.RedisCache.Get(r.key)
	}

	return
}

func (r *RedisKey) Set(data []byte) (err error) {

	if r.hash {
		err = redis.RedisCache.HMSet(r.key, r.hashid, data)
	} else {
		err = redis.RedisCache.Set(r.key, data)
	}

	if r.expire {
		return redis.RedisCache.Expire(r.key, 600)
	}

	return
}
<commit_msg>try hystrix on cache middleware<commit_after>package middleware

import (
	"github.com/afex/hystrix-go/hystrix"
	"github.com/gin-gonic/gin"
	"strings"

	"github.com/eirka/eirka-libs/redis"
)

var (
	RedisKeyIndex = make(map[string]RedisKey)
	RedisKeys     = []RedisKey{
		{base: "index", fieldcount: 1, hash: true, expire: false},
		{base: "thread", fieldcount: 2, hash: true, expire: false},
		{base: "tag", fieldcount: 2, hash: true, expire: true},
		{base: "image", fieldcount: 1, hash: true, expire: false},
		{base: "post", fieldcount: 2, hash: true, expire: false},
		{base: "tags", fieldcount: 1, hash: true, expire: false},
		{base: "directory", fieldcount: 1, hash: true, expire: false},
		{base: "new", fieldcount: 1, hash: false, expire: true},
		{base: "popular", fieldcount: 1, hash: false, expire: true},
		{base: "favorited", fieldcount: 1, hash: false, expire: true},
		{base: "tagtypes", fieldcount: 0, hash: false, expire: false},
		{base: "imageboards", fieldcount: 0, hash: false, expire: true},
	}
)

func init() {
	// key index map
	for _, key := range RedisKeys {
		RedisKeyIndex[key.base] = key
	}
}

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {
		var result []byte
		var err error

		// bool for analytics middleware
		c.Set("cached", false)

		// break cache if there is a query
		if c.Request.URL.RawQuery != "" {
			c.Next()
			return
		}

		// redis circuitbreaker
		err = hystrix.Do("cache", func() (err error) {

			// Trim leading / from path and split
			params := strings.Split(strings.Trim(c.Request.URL.Path, "/"), "/")

			// get the keyname
			key, ok := RedisKeyIndex[params[0]]
			if !ok {
				c.Next()
				return
			}

			// set the key minus the base
			key.SetKey(params[1:]...)

			result, err = key.Get()
			if err == redis.ErrCacheMiss {
				// go to the controller
				c.Next()

				// Check if there was an error from the controller
				_, controllerError := c.Get("controllerError")
				if controllerError {
					c.Abort()
					return
				}

				err = key.Set(c.MustGet("data").([]byte))
				if err != nil {
					c.Error(err)
					c.Abort()
					return
				}

			} else if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

			return

		}, func(e error) (err error) {
			c.Next()
			return
		})

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()
		return
	}

}

type RedisKey struct {
	base       string
	fieldcount int
	hash       bool
	expire     bool
	key        string
	hashid     string
}

func (r *RedisKey) SetKey(ids ...string) {

	if r.fieldcount == 0 {
		r.key = r.base
		return
	}

	// create our key
	r.key = strings.Join([]string{r.base, strings.Join(ids[:r.fieldcount], ":")}, ":")

	// get our hash id
	if r.hash {
		r.hashid = strings.Join(ids[r.fieldcount:], "")
	}

	return
}

func (r *RedisKey) Get() (result []byte, err error) {

	if r.hash {
		return redis.RedisCache.HGet(r.key, r.hashid)
	} else {
		return redis.RedisCache.Get(r.key)
	}

	return
}

func (r *RedisKey) Set(data []byte) (err error) {

	if r.hash {
		err = redis.RedisCache.HMSet(r.key, r.hashid, data)
	} else {
		err = redis.RedisCache.Set(r.key, data)
	}

	if r.expire {
		return redis.RedisCache.Expire(r.key, 600)
	}

	return
}
<|endoftext|>


<commit_before>// Package md2web contains the MD2Web trim.Application.
package md2web

import (
	"fmt"
	"io/ioutil"
	"net/http"
	gourl "net/url"
	"path/filepath"
	"strings"

	"github.com/jwowillo/pack"
	"github.com/jwowillo/trim/application"
	"github.com/jwowillo/trim/controller"
	"github.com/jwowillo/trim/request"
	"github.com/jwowillo/trim/response"
	"github.com/jwowillo/trim/url"
	"github.com/russross/blackfriday"
)

// MD2Web is a trim.Applications which turns directories of markdown files and
// folders into a website.
type MD2Web struct {
	*application.Web
}

// New creates a MD2Web excluding the provided files which has the given host.
func New(h, bf string, excs []string) *MD2Web {
	app := &MD2Web{Web: application.NewWeb()}
	app.RemoveAPI()
	app.ClearControllers()
	set := pack.NewHashSet(pack.StringHasher)
	for _, exc := range excs {
		set.Add(exc)
	}
	s := url.NewBuilder(h).
		SetSubdomain(app.Static().Subdomain()).
		SetPath(app.Static().BasePath())
	if err := app.AddController(newClientController(s, bf, set)); err != nil {
		panic(err)
	}
	return app
}

// NewDebug creates an MD2Web that doesn't cache which has the given host.
func NewDebug(h, bf string, excs []string) *MD2Web {
	cf := application.ClientDefault
	cf.CacheDuration = 0
	app := &MD2Web{
		Web: application.NewWebWithConfig(
			cf,
			application.APIDefault,
			application.StaticDefault,
		),
	}
	app.RemoveAPI()
	app.ClearControllers()
	set := pack.NewHashSet(pack.StringHasher)
	for _, exc := range excs {
		set.Add(exc)
	}
	s := url.NewBuilder(h).
		SetSubdomain(app.Static().Subdomain()).
		SetPath(app.Static().BasePath())
	if err := app.AddController(newClientController(s, bf, set)); err != nil {
		panic(err)
	}
	return app
}

// clientController which renders markdown page's based on request paths.
type clientController struct {
	controller.Bare
	staticBuilder *url.Builder
	baseFolder    string
	excludes      pack.Set
}

// newClientController creates a controller with the given template file and
// base folder.
func newClientController(
	sb *url.Builder,
	bf string,
	excs pack.Set,
) *clientController {
	excs.Add("static")
	excs.Add(".git")
	excs.Add(".gitignore")
	return &clientController{
		staticBuilder: sb,
		baseFolder:    bf,
		excludes:      excs,
	}
}

// Path of the clientController.
//
// Always a variable path which captures the entire path into the key
// 'fullName'.
func (c *clientController) Path() string {
	return "/:name*"
}

// Handle trim.Request by rendering the markdown page at the file name stored in
// the path.
func (c *clientController) Handle(req *request.Request) response.Response {
	fn := req.URL().Path()
	path := filepath.Join(c.baseFolder, buildPath(fn))
	path, err := gourl.QueryUnescape(path)
	hl, err := headerLinks(c.baseFolder, path, c.excludes)
	nl, err := navLinks(path, c.excludes)
	bs, err := content(path)
	proto := "http://"
	if req.TLS() != nil {
		proto = "https://"
	}
	static := c.staticBuilder.SetProtocol(proto).Build()
	args := pack.AnyMap{
		"title":       filepath.Base(fn),
		"static":      static,
		"headerLinks": hl,
		"navLinks":    nl,
		"content": strings.Replace(
			string(bs),
			"{{ static }}",
			static.String(),
			-1,
		),
	}
	if err != nil {
		args["headerLinks"] = map[string]string{"/": "/"}
		args["navLinks"] = nil
		args["content"] = fmt.Sprintf("%s couldn't be served.", fn)
		return response.NewTemplateFromString(
			Template,
			args,
			http.StatusInternalServerError,
		)
	}
	return response.NewTemplateFromString(Template, args, http.StatusOK)
}

// headerLinks are links to files along the provided path except what is in the
// provided set map mapped to their link text.
func headerLinks(bf, path string, excs pack.Set) ([]linkPair, error) {
	ls := []linkPair{linkPair{Real: "/", Fake: "/"}}
	working := ""
	for _, part := range strings.Split(filepath.Dir(path), "/") {
		if part == "." || part == bf {
			continue
		}
		working = filepath.Join(working, part)
		if excs.Contains(working) {
			return nil, fmt.Errorf("%s excluded", working)
		}
		if part == "main.md" {
			break
		}
		if strings.HasSuffix(part, ".md") {
			part = part[:len(part)-len(".md")]
		} else {
			part += "/"
		}
		ls = append(ls, linkPair{Real: "/" + working + "/", Fake: part})
	}
	return ls, nil
}

// navLinks are links to adjacent markdown files and folders to the provided
// path except what is in the excluded provided set mapped to their link text.
//
// Returns an error if the directory of the given path can't be read.
func navLinks(path string, excs pack.Set) ([]linkPair, error) {
	fs, err := ioutil.ReadDir(filepath.Dir(path))
	if err != nil {
		return nil, err
	}
	var ls []linkPair
	for _, f := range fs {
		fn := f.Name()
		if excs.Contains(fn) || excs.Contains(filepath.Base(fn)) {
			continue
		}
		key := f.Name()
		switch mode := f.Mode(); {
		case mode.IsDir():
			key = key + "/"
		case mode.IsRegular():
			if !strings.HasSuffix(fn, ".md") {
				continue
			}
			if fn == "main.md" {
				continue
			}
		}
		if strings.HasSuffix(key, ".md") {
			key = key[:len(key)-len(".md")]
			fn = fn[:len(fn)-len(".md")]
		}
		ls = append(ls, linkPair{Real: key, Fake: fn})
	}
	return ls, nil
}

// content of file at path.
//
// Returns an error if the file isn't a markdown file.
func content(path string) ([]byte, error) {
	if filepath.Ext(path) != ".md" {
		return nil, fmt.Errorf("%s isn't a markdown file", path)
	}
	bs, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	return blackfriday.MarkdownCommon(bs), nil
}

// buildPath to markdown file represented by given name.
func buildPath(name string) string {
	path := "." + name
	if path == "" || path[len(path)-1] == '/' {
		path += "main"
	}
	path += ".md"
	return path
}

// Template file shown as page.
const Template = `
<!DOCTYPE html>

    <meta charset="utf-8">
    <title>{{ title }}</title>
    <link rel="icon" href="{{ static }}/favicon.png">
    <link href="https://fonts.googleapis.com/css?family=Droid+Sans|Droid+Sans+Mono" rel="stylesheet" defer>
    <meta name="viewport" content="width=device-width, initial-scale=1">

      	{% for p in headerLinks %}
      	  {{ p.Fake }}
      	{% endfor %}

        {% for p in navLinks %}
          {{ p.Fake }}
        {% endfor %}

        {{ content | safe }}

`

// linkPair is a pair of a real and a fake link.
type linkPair struct {
	Real, Fake string
}
<commit_msg>Make Fonts Scoped<commit_after>// Package md2web contains the MD2Web trim.Application.
package md2web

import (
	"fmt"
	"io/ioutil"
	"net/http"
	gourl "net/url"
	"path/filepath"
	"strings"

	"github.com/jwowillo/pack"
	"github.com/jwowillo/trim/application"
	"github.com/jwowillo/trim/controller"
	"github.com/jwowillo/trim/request"
	"github.com/jwowillo/trim/response"
	"github.com/jwowillo/trim/url"
	"github.com/russross/blackfriday"
)

// MD2Web is a trim.Applications which turns directories of markdown files and
// folders into a website.
type MD2Web struct {
	*application.Web
}

// New creates a MD2Web excluding the provided files which has the given host.
func New(h, bf string, excs []string) *MD2Web {
	app := &MD2Web{Web: application.NewWeb()}
	app.RemoveAPI()
	app.ClearControllers()
	set := pack.NewHashSet(pack.StringHasher)
	for _, exc := range excs {
		set.Add(exc)
	}
	s := url.NewBuilder(h).
		SetSubdomain(app.Static().Subdomain()).
		SetPath(app.Static().BasePath())
	if err := app.AddController(newClientController(s, bf, set)); err != nil {
		panic(err)
	}
	return app
}

// NewDebug creates an MD2Web that doesn't cache which has the given host.
func NewDebug(h, bf string, excs []string) *MD2Web {
	cf := application.ClientDefault
	cf.CacheDuration = 0
	app := &MD2Web{
		Web: application.NewWebWithConfig(
			cf,
			application.APIDefault,
			application.StaticDefault,
		),
	}
	app.RemoveAPI()
	app.ClearControllers()
	set := pack.NewHashSet(pack.StringHasher)
	for _, exc := range excs {
		set.Add(exc)
	}
	s := url.NewBuilder(h).
		SetSubdomain(app.Static().Subdomain()).
		SetPath(app.Static().BasePath())
	if err := app.AddController(newClientController(s, bf, set)); err != nil {
		panic(err)
	}
	return app
}

// clientController which renders markdown page's based on request paths.
type clientController struct {
	controller.Bare
	staticBuilder *url.Builder
	baseFolder    string
	excludes      pack.Set
}

// newClientController creates a controller with the given template file and
// base folder.
func newClientController(
	sb *url.Builder,
	bf string,
	excs pack.Set,
) *clientController {
	excs.Add("static")
	excs.Add(".git")
	excs.Add(".gitignore")
	return &clientController{
		staticBuilder: sb,
		baseFolder:    bf,
		excludes:      excs,
	}
}

// Path of the clientController.
//
// Always a variable path which captures the entire path into the key
// 'fullName'.
func (c *clientController) Path() string {
	return "/:name*"
}

// Handle trim.Request by rendering the markdown page at the file name stored in
// the path.
func (c *clientController) Handle(req *request.Request) response.Response {
	fn := req.URL().Path()
	path := filepath.Join(c.baseFolder, buildPath(fn))
	path, err := gourl.QueryUnescape(path)
	hl, err := headerLinks(c.baseFolder, path, c.excludes)
	nl, err := navLinks(path, c.excludes)
	bs, err := content(path)
	proto := "http://"
	if req.TLS() != nil {
		proto = "https://"
	}
	static := c.staticBuilder.SetProtocol(proto).Build()
	args := pack.AnyMap{
		"title":       filepath.Base(fn),
		"static":      static,
		"headerLinks": hl,
		"navLinks":    nl,
		"content": strings.Replace(
			string(bs),
			"{{ static }}",
			static.String(),
			-1,
		),
	}
	if err != nil {
		args["headerLinks"] = map[string]string{"/": "/"}
		args["navLinks"] = nil
		args["content"] = fmt.Sprintf("%s couldn't be served.", fn)
		return response.NewTemplateFromString(
			Template,
			args,
			http.StatusInternalServerError,
		)
	}
	return response.NewTemplateFromString(Template, args, http.StatusOK)
}

// headerLinks are links to files along the provided path except what is in the
// provided set map mapped to their link text.
func headerLinks(bf, path string, excs pack.Set) ([]linkPair, error) {
	ls := []linkPair{linkPair{Real: "/", Fake: "/"}}
	working := ""
	for _, part := range strings.Split(filepath.Dir(path), "/") {
		if part == "." || part == bf {
			continue
		}
		working = filepath.Join(working, part)
		if excs.Contains(working) {
			return nil, fmt.Errorf("%s excluded", working)
		}
		if part == "main.md" {
			break
		}
		if strings.HasSuffix(part, ".md") {
			part = part[:len(part)-len(".md")]
		} else {
			part += "/"
		}
		ls = append(ls, linkPair{Real: "/" + working + "/", Fake: part})
	}
	return ls, nil
}

// navLinks are links to adjacent markdown files and folders to the provided
// path except what is in the excluded provided set mapped to their link text.
//
// Returns an error if the directory of the given path can't be read.
func navLinks(path string, excs pack.Set) ([]linkPair, error) {
	fs, err := ioutil.ReadDir(filepath.Dir(path))
	if err != nil {
		return nil, err
	}
	var ls []linkPair
	for _, f := range fs {
		fn := f.Name()
		if excs.Contains(fn) || excs.Contains(filepath.Base(fn)) {
			continue
		}
		key := f.Name()
		switch mode := f.Mode(); {
		case mode.IsDir():
			key = key + "/"
		case mode.IsRegular():
			if !strings.HasSuffix(fn, ".md") {
				continue
			}
			if fn == "main.md" {
				continue
			}
		}
		if strings.HasSuffix(key, ".md") {
			key = key[:len(key)-len(".md")]
			fn = fn[:len(fn)-len(".md")]
		}
		ls = append(ls, linkPair{Real: key, Fake: fn})
	}
	return ls, nil
}

// content of file at path.
//
// Returns an error if the file isn't a markdown file.
func content(path string) ([]byte, error) {
	if filepath.Ext(path) != ".md" {
		return nil, fmt.Errorf("%s isn't a markdown file", path)
	}
	bs, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	return blackfriday.MarkdownCommon(bs), nil
}

// buildPath to markdown file represented by given name.
func buildPath(name string) string {
	path := "." + name
	if path == "" || path[len(path)-1] == '/' {
		path += "main"
	}
	path += ".md"
	return path
}

// Template file shown as page.
const Template = `
<!DOCTYPE html>

    <meta charset="utf-8">
    <title>{{ title }}</title>
    <link rel="icon" href="{{ static }}/favicon.png">
    <meta name="viewport" content="width=device-width, initial-scale=1">

      	{% for p in headerLinks %}
      	  {{ p.Fake }}
      	{% endfor %}

        {% for p in navLinks %}
          {{ p.Fake }}
        {% endfor %}

        {{ content | safe }}

`

// linkPair is a pair of a real and a fake link.
type linkPair struct {
	Real, Fake string
}
<|endoftext|>