<commit_before>package houndify

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"time"
)

const houndifyVoiceURL = "https://api.houndify.com:443/v1/audio"
const houndifyTextURL = "https://api.houndify.com:443/v1/text"

// Default user agent sent set by the SDK
const SDKUserAgent = "Go Houndify SDK"

type (
	// A Client holds the configuration and state, which is used for
	// sending all outgoing Houndify requests and appropriately saving their responses.
	Client struct {
		// The ClientID comes from the Houndify site.
		ClientID string
		// The ClientKey comes from the Houndify site.
		// Keep the key secret.
		ClientKey               string
		enableConversationState bool
		conversationState       interface{}
		// If Verbose is true, all data sent from the server is printed to stdout, unformatted and unparsed.
		// This includes partial transcripts, errors, HTTP headers details (status code, headers, etc.), and final response JSON.
		Verbose bool
	}
	// A TextRequest holds all the information needed to make a Houndify request.
	// Create one of these per request to send and use a Client to send it.
	TextRequest struct {
		// The text query, e.g. "what time is it in london"
		Query             string
		UserID            string
		RequestID         string
		RequestInfoFields map[string]interface{}
	}
	// A VoiceRequest holds all the information needed to make a Houndify request.
	// Create one of these per request to send and use a Client to send it.
	VoiceRequest struct {
		// Stream of audio in bytes. It must already be in correct encoding.
		// See the Houndify docs for details.
		AudioStream       io.Reader
		UserID            string
		RequestID         string
		RequestInfoFields map[string]interface{}
	}

	// all of the Hound server JSON messages have these basic fields
	houndServerMessage struct {
		Format  string `json:"Format"`
		Version string `json:"FormatVersion"`
	}
	houndServerPartialTranscript struct {
		houndServerMessage
		PartialTranscript string `json:"PartialTranscript"`
		DurationMS        int64  `json:"DurationMS"`
		Done              bool   `json:"Done"`
	}
)

// EnableConversationState enables conversation state for future queries
func (c *Client) EnableConversationState() {
	c.enableConversationState = true
}

// DisableConversationState disables conversation state for future queries
func (c *Client) DisableConversationState() {
	c.enableConversationState = false
}

// ClearConversationState removes, or "forgets", the current conversation state
func (c *Client) ClearConversationState() {
	var emptyConvState interface{}
	c.conversationState = emptyConvState
}

// GetConversationState returns the current conversation state, useful for saving
func (c *Client) GetConversationState() interface{} {
	return c.conversationState
}

// SetConversationState sets the conversation state, useful for resuming from a saved point
func (c *Client) SetConversationState(newState interface{}) {
	c.conversationState = newState
}

// TextSearch sends a text request and returns the body of the Hound server response.
//
// An error is returned if there is a failure to create the request, failure to
// connect, failure to parse the response, or failure to update the conversation
// state (if applicable).
func (c *Client) TextSearch(textReq TextRequest) (string, error) {
	// setup http request
	body := []byte(``)
	req, err := http.NewRequest("POST", houndifyTextURL+"?query="+url.PathEscape(textReq.Query), bytes.NewBuffer(body))
	if err != nil {
		return "", errors.New("failed to build http request: " + err.Error())
	}
	// auth headers
	req.Header.Set("User-Agent", SDKUserAgent)
	clientAuth, requestAuth, timestamp, err := generateAuthValues(c.ClientID, c.ClientKey, textReq.UserID, textReq.RequestID)
	if err != nil {
		return "", errors.New("failed to create auth headers: " + err.Error())
	}
	req.Header.Set("Hound-Request-Authentication", requestAuth)
	req.Header.Set("Hound-Client-Authentication", clientAuth)

	// optional language headers
	if val, ok := textReq.RequestInfoFields["InputLanguageEnglishName"]; ok {
		req.Header.Set("InputLanguageEnglishName", val.(string))
	}
	if val, ok := textReq.RequestInfoFields["InputLanguageIETFTag"]; ok {
		req.Header.Set("InputLanguageIETFTag", val.(string))
	}

	// conversation state
	if c.enableConversationState {
		textReq.RequestInfoFields["ConversationState"] = c.conversationState
	} else {
		var emptyConvState interface{}
		textReq.RequestInfoFields["ConversationState"] = emptyConvState
	}

	// request info json
	requestInfo, err := createRequestInfo(c.ClientID, textReq.RequestID, timestamp, textReq.RequestInfoFields)
	if err != nil {
		return "", errors.New("failed to create request info: " + err.Error())
	}
	requestInfoJSON, err := json.Marshal(requestInfo)
	if err != nil {
		return "", errors.New("failed to create request info: " + err.Error())
	}
	req.Header.Set("Hound-Request-Info", string(requestInfoJSON))

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return "", errors.New("failed to successfully run request: " + err.Error())
	}

	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", errors.New("failed to read body: " + err.Error())
	}
	defer resp.Body.Close()

	bodyStr := string(body)

	if c.Verbose {
		fmt.Println(resp.Proto, resp.StatusCode)
		fmt.Println("Headers: ", resp.Header)
		fmt.Println(bodyStr)
	}

	//don't try to parse out conversation state from a bad response
	if resp.StatusCode >= 400 {
		return bodyStr, errors.New("error response")
	}
	// update with new conversation state
	if c.enableConversationState {
		newConvState, err := parseConversationState(bodyStr)
		if err != nil {
			return bodyStr, errors.New("unable to parse new conversation state from response")
		}
		c.conversationState = newConvState
	}

	return bodyStr, nil
}

// VoiceSearch sends an audio request and returns the body of the Hound server response.
//
// The partialTranscriptChan parameter allows the caller to receive for PartialTranscripts
// while the Hound server is listening to the voice search. If partial transcripts are not
// needed, create a throwaway channel that listens and discards all the PartialTranscripts
// sent.
//
// An error is returned if there is a failure to create the request, failure to
// connect, failure to parse the response, or failure to update the conversation
// state (if applicable).
func (c *Client) VoiceSearch(voiceReq VoiceRequest, partialTranscriptChan chan PartialTranscript) (string, error) {
	// setup http request
	req, err := http.NewRequest("POST", houndifyVoiceURL, nil)
	if err != nil {
		return "", errors.New("failed to build http request: " + err.Error())
	}
	// auth headers
	req.Header.Set("User-Agent", SDKUserAgent)
	clientAuth, requestAuth, timestamp, err := generateAuthValues(c.ClientID, c.ClientKey, voiceReq.UserID, voiceReq.RequestID)
	if err != nil {
		return "", errors.New("failed to create auth headers: " + err.Error())
	}
	req.Header.Set("Hound-Request-Authentication", requestAuth)
	req.Header.Set("Hound-Client-Authentication", clientAuth)

	// optional language headers
	if val, ok := voiceReq.RequestInfoFields["InputLanguageEnglishName"]; ok {
		req.Header.Set("InputLanguageEnglishName", val.(string))
	}
	if val, ok := voiceReq.RequestInfoFields["InputLanguageIETFTag"]; ok {
		req.Header.Set("InputLanguageIETFTag", val.(string))
	}

	// conversation state
	if c.enableConversationState {
		voiceReq.RequestInfoFields["ConversationState"] = c.conversationState
	} else {
		var emptyConvState interface{}
		voiceReq.RequestInfoFields["ConversationState"] = emptyConvState
	}

	// request info json
	requestInfo, err := createRequestInfo(c.ClientID, voiceReq.RequestID, timestamp, voiceReq.RequestInfoFields)
	if err != nil {
		return "", errors.New("failed to create request info: " + err.Error())
	}
	requestInfoJSON, err := json.Marshal(requestInfo)
	if err != nil {
		return "", errors.New("failed to create request info: " + err.Error())
	}
	req.Header.Set("Hound-Request-Info", string(requestInfoJSON))

	req.Body = ioutil.NopCloser(voiceReq.AudioStream)
	client := &http.Client{}

	// send the request

	resp, err := client.Do(req)
	if err != nil {
		return "", errors.New("failed to successfully run request: " + err.Error())
	}

	if c.Verbose {
		fmt.Println(resp.Proto, resp.StatusCode)
		fmt.Println("Headers: ", resp.Header)
	}

	// partial transcript parsing

	scanner := bufio.NewScanner(resp.Body)
	var line string
	for scanner.Scan() {
		line = scanner.Text()
		if c.Verbose {
			fmt.Println(line)
		}
		if line == "" {
			continue
		}
		if _, convertErr := strconv.Atoi(line); convertErr == nil {
			// this is an integer, so one of the ObjectByteCountPrefixes, skip it
			continue
		}
		// attempt to parse incoming json into partial transcript
		incoming := houndServerPartialTranscript{}
		if err := json.Unmarshal([]byte(line), &incoming); err != nil {
			fmt.Println("fail reading hound server message")
			continue
		}
		if incoming.Format == "HoundVoiceQueryPartialTranscript" || incoming.Format == "SoundHoundVoiceSearchParialTranscript" {
			// convert from houndify server's struct to SDK's simplified struct
			partialDuration, err := time.ParseDuration(fmt.Sprintf("%d", incoming.DurationMS) + "ms")
			if err != nil {
				fmt.Println("failed reading the time in partial transcript")
				continue
			}
			go func() {
				partialTranscriptChan <- PartialTranscript{
					Message:  incoming.PartialTranscript,
					Duration: partialDuration,
					Done:     incoming.Done,
				}
			}()
		} else if incoming.Format == "SoundHoundVoiceSearchResult" {
			// it wasn't actually a partial transcript, it was a final message with everything
			// we're done with partial transcripts now
			break
		}
	}
	close(partialTranscriptChan)

	body := line
	defer resp.Body.Close()

	// update with new conversation state
	if c.enableConversationState {
		newConvState, err := parseConversationState(string(body))
		if err != nil {
			return string(body), errors.New("unable to parse new conversation state from response")
		}
		c.conversationState = newConvState
	}

	return body, nil
}
<commit_msg>Crash fixes: (#7)<commit_after>package houndify

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"
)

const houndifyVoiceURL = "https://api.houndify.com:443/v1/audio"
const houndifyTextURL = "https://api.houndify.com:443/v1/text"

// Default user agent sent set by the SDK
const SDKUserAgent = "Go Houndify SDK"

type (
	// A Client holds the configuration and state, which is used for
	// sending all outgoing Houndify requests and appropriately saving their responses.
	Client struct {
		// The ClientID comes from the Houndify site.
		ClientID string
		// The ClientKey comes from the Houndify site.
		// Keep the key secret.
		ClientKey               string
		enableConversationState bool
		conversationState       interface{}
		// If Verbose is true, all data sent from the server is printed to stdout, unformatted and unparsed.
		// This includes partial transcripts, errors, HTTP headers details (status code, headers, etc.), and final response JSON.
		Verbose bool
	}
	// A TextRequest holds all the information needed to make a Houndify request.
	// Create one of these per request to send and use a Client to send it.
	TextRequest struct {
		// The text query, e.g. "what time is it in london"
		Query             string
		UserID            string
		RequestID         string
		RequestInfoFields map[string]interface{}
	}
	// A VoiceRequest holds all the information needed to make a Houndify request.
	// Create one of these per request to send and use a Client to send it.
	VoiceRequest struct {
		// Stream of audio in bytes. It must already be in correct encoding.
		// See the Houndify docs for details.
		AudioStream       io.Reader
		UserID            string
		RequestID         string
		RequestInfoFields map[string]interface{}
	}

	// all of the Hound server JSON messages have these basic fields
	houndServerMessage struct {
		Format  string `json:"Format"`
		Version string `json:"FormatVersion"`
	}
	houndServerPartialTranscript struct {
		houndServerMessage
		PartialTranscript string `json:"PartialTranscript"`
		DurationMS        int64  `json:"DurationMS"`
		Done              bool   `json:"Done"`
	}
)

// EnableConversationState enables conversation state for future queries
func (c *Client) EnableConversationState() {
	c.enableConversationState = true
}

// DisableConversationState disables conversation state for future queries
func (c *Client) DisableConversationState() {
	c.enableConversationState = false
}

// ClearConversationState removes, or "forgets", the current conversation state
func (c *Client) ClearConversationState() {
	var emptyConvState interface{}
	c.conversationState = emptyConvState
}

// GetConversationState returns the current conversation state, useful for saving
func (c *Client) GetConversationState() interface{} {
	return c.conversationState
}

// SetConversationState sets the conversation state, useful for resuming from a saved point
func (c *Client) SetConversationState(newState interface{}) {
	c.conversationState = newState
}

// TextSearch sends a text request and returns the body of the Hound server response.
//
// An error is returned if there is a failure to create the request, failure to
// connect, failure to parse the response, or failure to update the conversation
// state (if applicable).
func (c *Client) TextSearch(textReq TextRequest) (string, error) {
	// setup http request
	body := []byte(``)
	req, err := http.NewRequest("POST", houndifyTextURL+"?query="+url.PathEscape(textReq.Query), bytes.NewBuffer(body))
	if err != nil {
		return "", errors.New("failed to build http request: " + err.Error())
	}
	// auth headers
	req.Header.Set("User-Agent", SDKUserAgent)
	clientAuth, requestAuth, timestamp, err := generateAuthValues(c.ClientID, c.ClientKey, textReq.UserID, textReq.RequestID)
	if err != nil {
		return "", errors.New("failed to create auth headers: " + err.Error())
	}
	req.Header.Set("Hound-Request-Authentication", requestAuth)
	req.Header.Set("Hound-Client-Authentication", clientAuth)

	// optional language headers
	if val, ok := textReq.RequestInfoFields["InputLanguageEnglishName"]; ok {
		req.Header.Set("InputLanguageEnglishName", val.(string))
	}
	if val, ok := textReq.RequestInfoFields["InputLanguageIETFTag"]; ok {
		req.Header.Set("InputLanguageIETFTag", val.(string))
	}

	// conversation state
	if c.enableConversationState {
		textReq.RequestInfoFields["ConversationState"] = c.conversationState
	} else {
		var emptyConvState interface{}
		textReq.RequestInfoFields["ConversationState"] = emptyConvState
	}

	// request info json
	requestInfo, err := createRequestInfo(c.ClientID, textReq.RequestID, timestamp, textReq.RequestInfoFields)
	if err != nil {
		return "", errors.New("failed to create request info: " + err.Error())
	}
	requestInfoJSON, err := json.Marshal(requestInfo)
	if err != nil {
		return "", errors.New("failed to create request info: " + err.Error())
	}
	req.Header.Set("Hound-Request-Info", string(requestInfoJSON))

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return "", errors.New("failed to successfully run request: " + err.Error())
	}

	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", errors.New("failed to read body: " + err.Error())
	}
	defer resp.Body.Close()

	bodyStr := string(body)

	if c.Verbose {
		fmt.Println(resp.Proto, resp.StatusCode)
		fmt.Println("Headers: ", resp.Header)
		fmt.Println(bodyStr)
	}

	//don't try to parse out conversation state from a bad response
	if resp.StatusCode >= 400 {
		return bodyStr, errors.New("error response")
	}
	// update with new conversation state
	if c.enableConversationState {
		newConvState, err := parseConversationState(bodyStr)
		if err != nil {
			return bodyStr, errors.New("unable to parse new conversation state from response")
		}
		c.conversationState = newConvState
	}

	return bodyStr, nil
}

// VoiceSearch sends an audio request and returns the body of the Hound server response.
//
// The partialTranscriptChan parameter allows the caller to receive for PartialTranscripts
// while the Hound server is listening to the voice search. If partial transcripts are not
// needed, create a throwaway channel that listens and discards all the PartialTranscripts
// sent.
//
// An error is returned if there is a failure to create the request, failure to
// connect, failure to parse the response, or failure to update the conversation
// state (if applicable).
func (c *Client) VoiceSearch(voiceReq VoiceRequest, partialTranscriptChan chan PartialTranscript) (string, error) {
	// setup http request
	req, err := http.NewRequest("POST", houndifyVoiceURL, nil)
	if err != nil {
		return "", errors.New("failed to build http request: " + err.Error())
	}
	// auth headers
	req.Header.Set("User-Agent", SDKUserAgent)
	clientAuth, requestAuth, timestamp, err := generateAuthValues(c.ClientID, c.ClientKey, voiceReq.UserID, voiceReq.RequestID)
	if err != nil {
		return "", errors.New("failed to create auth headers: " + err.Error())
	}
	req.Header.Set("Hound-Request-Authentication", requestAuth)
	req.Header.Set("Hound-Client-Authentication", clientAuth)

	// optional language headers
	if val, ok := voiceReq.RequestInfoFields["InputLanguageEnglishName"]; ok {
		req.Header.Set("InputLanguageEnglishName", val.(string))
	}
	if val, ok := voiceReq.RequestInfoFields["InputLanguageIETFTag"]; ok {
		req.Header.Set("InputLanguageIETFTag", val.(string))
	}

	// conversation state
	if c.enableConversationState {
		voiceReq.RequestInfoFields["ConversationState"] = c.conversationState
	} else {
		var emptyConvState interface{}
		voiceReq.RequestInfoFields["ConversationState"] = emptyConvState
	}

	// request info json
	requestInfo, err := createRequestInfo(c.ClientID, voiceReq.RequestID, timestamp, voiceReq.RequestInfoFields)
	if err != nil {
		return "", errors.New("failed to create request info: " + err.Error())
	}
	requestInfoJSON, err := json.Marshal(requestInfo)
	if err != nil {
		return "", errors.New("failed to create request info: " + err.Error())
	}
	req.Header.Set("Hound-Request-Info", string(requestInfoJSON))

	req.Body = ioutil.NopCloser(voiceReq.AudioStream)
	client := &http.Client{}

	// send the request

	resp, err := client.Do(req)
	if err != nil {
		return "", errors.New("failed to successfully run request: " + err.Error())
	}

	if c.Verbose {
		fmt.Println(resp.Proto, resp.StatusCode)
		fmt.Println("Headers: ", resp.Header)
	}

	// partial transcript parsing

	//so the partial transcript channel doesn't get closed before all transcripts are sent
	partialChanWait := sync.WaitGroup{}
	reader := bufio.NewReader(resp.Body)
	var line string
	for {
		bytes, err := reader.ReadBytes('\n')
		line = strings.TrimSpace(string(bytes))
		if c.Verbose {
			fmt.Println(line)
		}
		if err != nil {
			if err != io.EOF {
				fmt.Println(err)
				return "", errors.New("error reading Houndify server response")
			}
			//EOF means this line must be the final response, done with partial transcripts
			break
		}
		if line == "" {
			continue
		}
		if _, convertErr := strconv.Atoi(line); convertErr == nil {
			// this is an integer, so one of the ObjectByteCountPrefixes, skip it
			continue
		}
		// attempt to parse incoming json into partial transcript
		incoming := houndServerPartialTranscript{}
		if err := json.Unmarshal([]byte(line), &incoming); err != nil {
			fmt.Println("fail reading hound server message")
			continue
		}
		if incoming.Format == "HoundVoiceQueryPartialTranscript" || incoming.Format == "SoundHoundVoiceSearchParialTranscript" {
			// convert from houndify server's struct to SDK's simplified struct
			partialDuration, err := time.ParseDuration(fmt.Sprintf("%d", incoming.DurationMS) + "ms")
			if err != nil {
				fmt.Println("failed reading the time in partial transcript")
				continue
			}
			partialChanWait.Add(1)
			go func() {
				partialTranscriptChan <- PartialTranscript{
					Message:  incoming.PartialTranscript,
					Duration: partialDuration,
					Done:     incoming.Done,
				}
				partialChanWait.Done()
			}()
			continue
		}
		if incoming.Format == "SoundHoundVoiceSearchResult" {
			//this line is the final response, done with partial transcripts
			break
		}
	}
	go func() {
		//don't close the open partial transcript channel
		partialChanWait.Wait()
		close(partialTranscriptChan)
	}()

	bodyStr := line
	defer resp.Body.Close()

	//don't try to parse out conversation state from a bad response
	if resp.StatusCode >= 400 {
		return bodyStr, errors.New("error response")
	}
	// update with new conversation state
	if c.enableConversationState {
		newConvState, err := parseConversationState(bodyStr)
		if err != nil {
			return bodyStr, errors.New("unable to parse new conversation state from response")
		}
		c.conversationState = newConvState
	}

	return bodyStr, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Mark Wolfe. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package buildkite

import (
	"encoding/base64"
	"fmt"
	"net/http"
	"time"
)

// TokenAuthTransport manages injection of the API token for each request
type TokenAuthTransport struct {
	APIToken string
	Debug    bool
}

// RoundTrip invoked each time a request is made
func (t TokenAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIToken))
	ts := time.Now()
	res, err := http.DefaultTransport.RoundTrip(req)
	if t.Debug {
		fmt.Printf("DEBUG uri = %s time = %s\n", req.URL, time.Now().Sub(ts))
	}
	return res, err
}

// Client builds a new http client.
func (t *TokenAuthTransport) Client() *http.Client {
	return &http.Client{Transport: t}
}

// NewTokenConfig configure authentication using an API token
func NewTokenConfig(apiToken string, debug bool) (*TokenAuthTransport, error) {
	if apiToken == "" {
		return nil, fmt.Errorf("Invalid token, empty string supplied")
	}
	return &TokenAuthTransport{APIToken: apiToken, Debug: debug}, nil
}

// BasicAuthTransport manages injection of the authorization header
type BasicAuthTransport struct {
	Username string
	Password string
}

// RoundTrip invoked each time a request is made
func (bat BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	req.Header.Set("Authorization", fmt.Sprintf("Basic %s",
		base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s",
			bat.Username, bat.Password)))))
	return http.DefaultTransport.RoundTrip(req)
}

// Client builds a new http client.
func (bat *BasicAuthTransport) Client() *http.Client {
	return &http.Client{Transport: bat}
}

// NewBasicConfig configure authentication using the supplied credentials
func NewBasicConfig(username string, password string) (*BasicAuthTransport, error) {
	if username == "" {
		return nil, fmt.Errorf("Invalid username, empty string supplied")
	}
	if password == "" {
		return nil, fmt.Errorf("Invalid password, empty string supplied")
	}
	return &BasicAuthTransport{username, password}, nil
}
<commit_msg>Tweaked debug output.<commit_after>// Copyright 2014 Mark Wolfe. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package buildkite

import (
	"encoding/base64"
	"fmt"
	"net/http"
	"time"
)

// TokenAuthTransport manages injection of the API token for each request
type TokenAuthTransport struct {
	APIToken string
	Debug    bool
}

// RoundTrip invoked each time a request is made
func (t TokenAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIToken))
	ts := time.Now()
	res, err := http.DefaultTransport.RoundTrip(req)
	if t.Debug {
		fmt.Printf("DEBUG uri=%s time=%s\n", req.URL, time.Now().Sub(ts))
	}
	return res, err
}

// Client builds a new http client.
func (t *TokenAuthTransport) Client() *http.Client {
	return &http.Client{Transport: t}
}

// NewTokenConfig configure authentication using an API token
func NewTokenConfig(apiToken string, debug bool) (*TokenAuthTransport, error) {
	if apiToken == "" {
		return nil, fmt.Errorf("Invalid token, empty string supplied")
	}
	return &TokenAuthTransport{APIToken: apiToken, Debug: debug}, nil
}

// BasicAuthTransport manages injection of the authorization header
type BasicAuthTransport struct {
	Username string
	Password string
}

// RoundTrip invoked each time a request is made
func (bat BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	req.Header.Set("Authorization", fmt.Sprintf("Basic %s",
		base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s",
			bat.Username, bat.Password)))))
	return http.DefaultTransport.RoundTrip(req)
}

// Client builds a new http client.
func (bat *BasicAuthTransport) Client() *http.Client {
	return &http.Client{Transport: bat}
}

// NewBasicConfig configure authentication using the supplied credentials
func NewBasicConfig(username string, password string) (*BasicAuthTransport, error) {
	if username == "" {
		return nil, fmt.Errorf("Invalid username, empty string supplied")
	}
	if password == "" {
		return nil, fmt.Errorf("Invalid password, empty string supplied")
	}
	return &BasicAuthTransport{username, password}, nil
}
<|endoftext|>


<commit_before>/*
   Copyright (c) 2014, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package monitor_test

import (
	"github.com/percona/cloud-protocol/proto"
	"github.com/percona/percona-agent/mrms/monitor"
	"github.com/percona/percona-agent/pct"
	"github.com/percona/percona-agent/test/mock"
	. "launchpad.net/gocheck"
	"testing"
	"time"
)

func Test(t *testing.T) { TestingT(t) }

/////////////////////////////////////////////////////////////////////////////
// Test Suite
/////////////////////////////////////////////////////////////////////////////

type TestSuite struct {
	nullmysql *mock.NullMySQL
	logChan   chan *proto.LogEntry
	logger    *pct.Logger
}

var _ = Suite(&TestSuite{})

func (s *TestSuite) SetUpSuite(t *C) {
	s.nullmysql = mock.NewNullMySQL()
	s.logChan = make(chan *proto.LogEntry, 1000)
	s.logger = pct.NewLogger(s.logChan, "mrms-monitor-test")
}

func (s *TestSuite) TestStartStop(t *C) {
	firstUptime := make(chan bool, 1)
	mockConn := &mock.ConnectorMock{
		ConnectMock: func(tries uint) error {
			return nil
		},
		CloseMock: func() {
		},
		UptimeMock: func() int64 {
			firstUptime <- true
			return 10
		},
	}
	mockConnFactory := &mock.ConnectionFactory{
		Conn: mockConn,
	}
	m := monitor.NewMonitor(s.logger, mockConnFactory)
	dsn := "fake:dsn@tcp(127.0.0.1:3306)/?parseTime=true"

	/**
	 * Register new subscriber
	 */
	subChan, err := m.Add(dsn)
	t.Assert(err, IsNil)

	/**
	 * Start MRMS
	 */
	err = m.Start()
	t.Assert(err, IsNil)

	select {
	case <-firstUptime:
	case <-time.After(1 * time.Second):
		t.Errorf("MRMS didn't checked uptime upon startup")
	}

	// Let's imitate MySQL restart
	mockConn.UptimeMock = func() int64 {
		return 5
	}

	// After max 1 second it should notify subscriber about MySQL restart
	var notified bool
	select {
	case notified = <-subChan:
	case <-time.After(1 * time.Second):
	}
	t.Assert(notified, Equals, true, Commentf("MySQL was restarted but MRMS didn't notify subscribers"))

	/**
	 * Stop MRMS
	 */
	err = m.Stop()
	t.Assert(err, IsNil)

	// Let's imitate MySQL restart
	mockConn.UptimeMock = func() int64 {
		return 1
	}

	// After stopping service it should not notify subscribers anymore
	time.Sleep(2 * time.Second)
	select {
	case notified = <-subChan:
	default:
	}
	t.Assert(notified, Equals, true, Commentf("MRMS notified subscribers after being stopped"))
}

func (s *TestSuite) TestNotifications(t *C) {
	mockConn := &mock.ConnectorMock{
		ConnectMock: func(tries uint) error {
			return nil
		},
		CloseMock: func() {
		},
		UptimeMock: func() int64 {
			return time.Now().Unix()
		},
	}
	mockConnFactory := &mock.ConnectionFactory{
		Conn: mockConn,
	}
	m := monitor.NewMonitor(s.logger, mockConnFactory)
	dsn := "fake:dsn@tcp(127.0.0.1:3306)/?parseTime=true"

	/**
	 * Register new subscriber
	 */
	subChan, err := m.Add(dsn)
	t.Assert(err, IsNil)

	/**
	 * MRMS should not send notification after first check for given dsn
	 */
	var notified bool
	select {
	case notified = <-subChan:
	default:
	}
	t.Assert(notified, Equals, false, Commentf("MySQL was not restarted (first check of MySQL server), but MRMS notified subscribers"))

	/**
	 * If MySQL was restarted then MRMS should notify subscriber
	 */
	mockConn.UptimeMock = func() int64 {
		return 0
	}
	m.Check()
	notified = false
	select {
	case notified = <-subChan:
	default:
	}
	t.Assert(notified, Equals, true, Commentf("MySQL was restarted, but MRMS didn't notify subscribers"))

	/**
	 * If MySQL was not restarted then MRMS should not notify subscriber
	 */
	mockConn.UptimeMock = func() int64 {
		return 2
	}
	m.Check()
	notified = false
	select {
	case notified = <-subChan:
	default:
	}
	t.Assert(notified, Equals, false, Commentf("MySQL was not restarted, but MRMS notified subscribers"))

	/**
	 * Now let's imitate MySQL server restart and let's wait 3 seconds before next check.
	 * Since MySQL server was restarted and we waited 3s then uptime=3s
	 * which is higher than last registered uptime=2s
	 *
	 * However we expect in this test that this is properly detected as MySQL restart
	 * and the MRMS notifies subscribers
	 */
	waitTime := int64(3)
	time.Sleep(time.Duration(waitTime) * time.Second)
	mockConn.UptimeMock = func() int64 {
		return waitTime
	}
	m.Check()
	select {
	case notified = <-subChan:
	default:
	}
	t.Assert(notified, Equals, true, Commentf("MySQL was restarted (uptime overlaped last registered uptime), but MRMS didn't notify subscribers"))

	/**
	 * After removing subscriber MRMS should not notify it anymore about MySQL restarts
	 */
	mockConn.UptimeMock = func() int64 {
		return 0
	}
	m.Remove(dsn, subChan)
	m.Check()
	notified = false
	select {
	case notified = <-subChan:
	default:
	}
	t.Assert(notified, Equals, false, Commentf("Subscriber was removed but MRMS still notified it about MySQL restart"))
}
<commit_msg>PCT-637: better comments<commit_after>/*
   Copyright (c) 2014, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package monitor_test

import (
	"github.com/percona/cloud-protocol/proto"
	"github.com/percona/percona-agent/mrms/monitor"
	"github.com/percona/percona-agent/pct"
	"github.com/percona/percona-agent/test/mock"
	. "launchpad.net/gocheck"
	"testing"
	"time"
)

func Test(t *testing.T) { TestingT(t) }

/////////////////////////////////////////////////////////////////////////////
// Test Suite
/////////////////////////////////////////////////////////////////////////////

type TestSuite struct {
	nullmysql *mock.NullMySQL
	logChan   chan *proto.LogEntry
	logger    *pct.Logger
}

var _ = Suite(&TestSuite{})

func (s *TestSuite) SetUpSuite(t *C) {
	s.nullmysql = mock.NewNullMySQL()
	s.logChan = make(chan *proto.LogEntry, 1000)
	s.logger = pct.NewLogger(s.logChan, "mrms-monitor-test")
}

func (s *TestSuite) TestStartStop(t *C) {
	mockConn := &mock.ConnectorMock{
		ConnectMock: func(tries uint) error {
			return nil
		},
		CloseMock: func() {
		},
	}
	mockConnFactory := &mock.ConnectionFactory{
		Conn: mockConn,
	}
	m := monitor.NewMonitor(s.logger, mockConnFactory)
	dsn := "fake:dsn@tcp(127.0.0.1:3306)/?parseTime=true"

	/**
	 * Register new subscriber
	 */
	firstUptime := make(chan bool, 1)
	mockConn.UptimeMock = func() int64 {
		firstUptime <- true
		return 10 // Initial uptime
	}
	subChan, err := m.Add(dsn)
	t.Assert(err, IsNil)

	select {
	case <-firstUptime:
	case <-time.After(1 * time.Second):
		t.Errorf("MRMS didn't checked uptime after adding first subscriber")
	}

	/**
	 * Start MRMS
	 */
	err = m.Start()
	t.Assert(err, IsNil)

	// Let's imitate MySQL restart
	mockConn.UptimeMock = func() int64 {
		return 5 // Imitate MySQL restart by setting uptime to 5s (previously 10s)
	}

	// After max 1 second it should notify subscriber about MySQL restart
	var notified bool
	select {
	case notified = <-subChan:
	case <-time.After(1 * time.Second):
	}
	t.Assert(notified, Equals, true, Commentf("MySQL was restarted but MRMS didn't notify subscribers"))

	/**
	 * Stop MRMS
	 */
	err = m.Stop()
	t.Assert(err, IsNil)

	// Let's imitate MySQL restart
	mockConn.UptimeMock = func() int64 {
		return 1 // Imitate MySQL restart by setting uptime to 1s (previously 5s)
	}

	// After stopping service it should not notify subscribers anymore
	time.Sleep(2 * time.Second)
	select {
	case notified = <-subChan:
	default:
	}
	t.Assert(notified, Equals, true, Commentf("MRMS notified subscribers after being stopped"))
}

func (s *TestSuite) TestNotifications(t *C) {
	mockConn := &mock.ConnectorMock{
		ConnectMock: func(tries uint) error {
			return nil
		},
		CloseMock: func() {
		},
	}
	mockConnFactory := &mock.ConnectionFactory{
		Conn: mockConn,
	}
	m := monitor.NewMonitor(s.logger, mockConnFactory)
	dsn := "fake:dsn@tcp(127.0.0.1:3306)/?parseTime=true"

	/**
	 * Register new subscriber
	 */
	mockConn.UptimeMock = func() int64 {
		return 10 // Initial uptime
	}
	subChan, err := m.Add(dsn)
	t.Assert(err, IsNil)

	/**
	 * MRMS should not send notification after first check for given dsn
	 */
	var notified bool
	select {
	case notified = <-subChan:
	default:
	}
	t.Assert(notified, Equals, false, Commentf("MySQL was not restarted (first check of MySQL server), but MRMS notified subscribers"))

	/**
	 * If MySQL was restarted then MRMS should notify subscriber
	 */
	mockConn.UptimeMock = func() int64 {
		return 0 // imitate MySQL restart by returning 0s uptime
	}
	m.Check()
	notified = false
	select {
	case notified = <-subChan:
	default:
	}
	t.Assert(notified, Equals, true, Commentf("MySQL was restarted, but MRMS didn't notify subscribers"))

	/**
	 * If MySQL was not restarted then MRMS should not notify subscriber
	 */
	mockConn.UptimeMock = func() int64 {
		return 2 // 2s uptime is higher than previous 0s, this indicates MySQL was not restarted
	}
	m.Check()
	notified = false
	select {
	case notified = <-subChan:
	default:
	}
	t.Assert(notified, Equals, false, Commentf("MySQL was not restarted, but MRMS notified subscribers"))

	/**
	 * Now let's imitate MySQL server restart and let's wait 3 seconds before next check.
	 * Since MySQL server was restarted and we waited 3s then uptime=3s
	 * which is higher than last registered uptime=2s
	 *
	 * However we expect in this test that this is properly detected as MySQL restart
	 * and the MRMS notifies subscribers
	 */
	waitTime := int64(3)
	time.Sleep(time.Duration(waitTime) * time.Second)
	mockConn.UptimeMock = func() int64 {
		return waitTime
	}
	m.Check()
	select {
	case notified = <-subChan:
	default:
	}
	t.Assert(notified, Equals, true, Commentf("MySQL was restarted (uptime overlaped last registered uptime), but MRMS didn't notify subscribers"))

	/**
	 * After removing subscriber MRMS should not notify it anymore about MySQL restarts
	 */
	mockConn.UptimeMock = func() int64 {
		return 0 // imitate MySQL restart by returning 0s uptime
	}
	m.Remove(dsn, subChan)
	m.Check()
	notified = false
	select {
	case notified = <-subChan:
	default:
	}
	t.Assert(notified, Equals, false, Commentf("Subscriber was removed but MRMS still notified it about MySQL restart"))
}
<|endoftext|>


<commit_before>package bulk

import (
	"crypto/tls"
	"net"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/cloudfoundry-incubator/cf_http"
	"github.com/cloudfoundry-incubator/nsync/recipebuilder"
	"github.com/cloudfoundry-incubator/receptor"
	"github.com/cloudfoundry-incubator/runtime-schema/cc_messages"
	"github.com/cloudfoundry-incubator/runtime-schema/metric"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
)

const (
	syncDesiredLRPsDuration = metric.Duration("DesiredLRPSyncDuration")
)

//go:generate counterfeiter -o fakes/fake_recipe_builder.go . RecipeBuilder
type RecipeBuilder interface {
	Build(*cc_messages.DesireAppRequestFromCC) (*receptor.DesiredLRPCreateRequest, error)
}

type Processor struct {
	receptorClient  receptor.Client
	pollingInterval time.Duration
	domainTTL       time.Duration
	bulkBatchSize   uint
	skipCertVerify  bool
	logger          lager.Logger
	fetcher         Fetcher
	builder         RecipeBuilder
	clock           clock.Clock
}

func NewProcessor(
	receptorClient receptor.Client,
	pollingInterval time.Duration,
	domainTTL time.Duration,
	bulkBatchSize uint,
	skipCertVerify bool,
	logger lager.Logger,
	fetcher Fetcher,
	builder RecipeBuilder,
	clock clock.Clock,
) *Processor {
	return &Processor{
		receptorClient:  receptorClient,
		pollingInterval: pollingInterval,
		domainTTL:       domainTTL,
		bulkBatchSize:   bulkBatchSize,
		skipCertVerify:  skipCertVerify,
		logger:          logger,
		fetcher:         fetcher,
		builder:         builder,
		clock:           clock,
	}
}

func (p *Processor) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
	close(ready)

	httpClient := cf_http.NewClient()
	httpClient.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: p.skipCertVerify,
			MinVersion:         tls.VersionTLS10,
		},
	}

	timer := p.clock.NewTimer(p.pollingInterval)
	stop := p.sync(signals, httpClient)

	for {
		if stop {
			return nil
		}

		select {
		case <-signals:
			return nil
		case <-timer.C():
			stop = p.sync(signals, httpClient)
			timer.Reset(p.pollingInterval)
		}
	}
}

func (p *Processor) sync(signals <-chan os.Signal, httpClient *http.Client) bool {
	start := p.clock.Now()
	defer func() {
		duration := p.clock.Now().Sub(start)
		syncDesiredLRPsDuration.Send(duration)
	}()

	logger := p.logger.Session("sync")

	existing, err := p.getDesiredLRPs(logger)
	if err != nil {
		return false
	}

	differ := NewDiffer(existing)

	cancel := make(chan struct{})

	fingerprints, fingerprintErrors := p.fetcher.FetchFingerprints(
		logger,
		cancel,
		httpClient,
	)

	diffErrors := differ.Diff(
		logger,
		cancel,
		fingerprints,
	)

	missingApps, missingAppsErrors := p.fetcher.FetchDesiredApps(
		logger,
		cancel,
		httpClient,
		differ.Missing(),
	)

	createErrors := p.createMissingDesiredLRPs(logger, cancel, missingApps)

	staleApps, staleAppErrors := p.fetcher.FetchDesiredApps(
		logger,
		cancel,
		httpClient,
		differ.Stale(),
	)

	updateErrors := p.updateStaleDesiredLRPs(logger, cancel, staleApps)

	bumpFreshness := true
	success := true

	fingerprintErrors, fingerprintErrorCount := countErrors(fingerprintErrors)

	errors := mergeErrors(
		fingerprintErrors,
		diffErrors,
		missingAppsErrors,
		staleAppErrors,
		createErrors,
		updateErrors,
	)

process_loop:
	for {
		select {
		case err, open := <-errors:
			if err != nil {
				bumpFreshness = false
			}
			if !open {
				break process_loop
			}
		case <-signals:
			close(cancel)
			return true
		}
	}

	if <-fingerprintErrorCount != 0 {
		logger.Error("failed-to-fetch-all-cc-fingerprints", nil)
		success = false
	}

	if success {
		deleteList := <-differ.Deleted()
		p.deleteExcess(logger, cancel, deleteList)
	}

	if bumpFreshness && success {
		err = p.receptorClient.UpsertDomain(recipebuilder.LRPDomain, p.domainTTL)
		if err != nil {
			logger.Error("failed-to-upsert-domain", err)
		}
	}

	return false
}

func (p *Processor) createMissingDesiredLRPs(
	logger lager.Logger,
	cancel <-chan struct{},
	missing <-chan []cc_messages.DesireAppRequestFromCC,
) <-chan error {
	logger = logger.Session("create-missing-desired-lrps")

	errc := make(chan error, 1)

	go func() {
		defer close(errc)

		for {
			var desireAppRequests []cc_messages.DesireAppRequestFromCC

			select {
			case <-cancel:
				return

			case selected, open := <-missing:
				if !open {
					return
				}

				desireAppRequests = selected
			}

			logger.Info("processing-batch", lager.Data{"size": len(desireAppRequests)})

			for _, desireAppRequest := range desireAppRequests {
				createReq, err := p.builder.Build(&desireAppRequest)
				if err != nil {
					logger.Error("failed-to-build-create-desired-lrp-request", err, lager.Data{
						"desire-app-request": desireAppRequest,
					})
					errc <- err
					continue
				}

				err = p.receptorClient.CreateDesiredLRP(*createReq)
				if err != nil {
					logger.Error("failed-to-create-desired-lrp", err, lager.Data{
						"create-request": createReq,
					})
					errc <- err
					continue
				}
			}
		}
	}()

	return errc
}

func (p *Processor) updateStaleDesiredLRPs(
	logger lager.Logger,
	cancel <-chan struct{},
	stale <-chan []cc_messages.DesireAppRequestFromCC,
) <-chan error {
	logger = logger.Session("update-stale-desired-lrps")

	errc := make(chan error, 1)

	go func() {
		defer close(errc)

		for {
			var staleAppRequests []cc_messages.DesireAppRequestFromCC

			select {
			case <-cancel:
				return

			case selected, open := <-stale:
				if !open {
					return
				}

				staleAppRequests = selected
			}

			logger.Info("processing-batch", lager.Data{"size": len(staleAppRequests)})

			for _, desireAppRequest := range staleAppRequests {
				updateReq := receptor.DesiredLRPUpdateRequest{}
				updateReq.Instances = &desireAppRequest.NumInstances
				updateReq.Annotation = &desireAppRequest.ETag
				updateReq.Routes = cc_messages.NewRoutingInfo(desireAppRequest.Hostnames, recipebuilder.DefaultPort)

				err := p.receptorClient.UpdateDesiredLRP(desireAppRequest.ProcessGuid, updateReq)
				if err != nil {
					logger.Error("failed-to-update-stale-lrp", err, lager.Data{
						"update-request": updateReq,
					})
					errc <- err
					continue
				}
			}
		}
	}()

	return errc
}

func (p *Processor) getDesiredLRPs(logger lager.Logger) ([]receptor.DesiredLRPResponse, error) {
	logger.Info("getting-desired-lrps-from-bbs")

	existing, err := p.receptorClient.DesiredLRPsByDomain(recipebuilder.LRPDomain)
	if err != nil {
		logger.Error("failed-to-get-desired-lrps", err)
		return nil, err
	}

	logger.Info("got-desired-lrps-from-bbs", lager.Data{"count": len(existing)})
	return existing, nil
}

func (p *Processor) deleteExcess(logger lager.Logger, cancel <-chan struct{}, excess []string) {
	logger = logger.Session("delete-excess")

	logger.Info(
		"processing-batch",
		lager.Data{"size": len(excess)},
	)

	for _, deleteGuid := range excess {
		err := p.receptorClient.DeleteDesiredLRP(deleteGuid)
		if err != nil {
			logger.Error(
				"failed-to-delete-desired-lrp",
				err,
				lager.Data{"delete-request": deleteGuid},
			)
		}
	}
}

func countErrors(source <-chan error) (<-chan error, <-chan int) {
	count := make(chan int, 1)
	dest := make(chan error, 1)
	var errorCount int

	wg := sync.WaitGroup{}
	wg.Add(1)

	go func() {
		for e := range source {
			errorCount++
			dest <- e
		}

		close(dest)
		wg.Done()
	}()

	go func() {
		wg.Wait()

		count <- errorCount
		close(count)
	}()

	return dest, count
}

func mergeErrors(channels ...<-chan error) <-chan error {
	out := make(chan error)
	wg := sync.WaitGroup{}

	for _, ch := range channels {
		wg.Add(1)

		go func(c <-chan error) {
			for e := range c {
				out <- e
			}
			wg.Done()
		}(ch)
	}

	go func() {
		wg.Wait()
		close(out)
	}()

	return out
}
<commit_msg>log when bumping or not bumping freshness<commit_after>package bulk

import (
	"crypto/tls"
	"net"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/cloudfoundry-incubator/cf_http"
	"github.com/cloudfoundry-incubator/nsync/recipebuilder"
	"github.com/cloudfoundry-incubator/receptor"
	"github.com/cloudfoundry-incubator/runtime-schema/cc_messages"
	"github.com/cloudfoundry-incubator/runtime-schema/metric"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
)

const (
	syncDesiredLRPsDuration = metric.Duration("DesiredLRPSyncDuration")
)

//go:generate counterfeiter -o fakes/fake_recipe_builder.go . RecipeBuilder
type RecipeBuilder interface {
	Build(*cc_messages.DesireAppRequestFromCC) (*receptor.DesiredLRPCreateRequest, error)
}

type Processor struct {
	receptorClient  receptor.Client
	pollingInterval time.Duration
	domainTTL       time.Duration
	bulkBatchSize   uint
	skipCertVerify  bool
	logger          lager.Logger
	fetcher         Fetcher
	builder         RecipeBuilder
	clock           clock.Clock
}

func NewProcessor(
	receptorClient receptor.Client,
	pollingInterval time.Duration,
	domainTTL time.Duration,
	bulkBatchSize uint,
	skipCertVerify bool,
	logger lager.Logger,
	fetcher Fetcher,
	builder RecipeBuilder,
	clock clock.Clock,
) *Processor {
	return &Processor{
		receptorClient:  receptorClient,
		pollingInterval: pollingInterval,
		domainTTL:       domainTTL,
		bulkBatchSize:   bulkBatchSize,
		skipCertVerify:  skipCertVerify,
		logger:          logger,
		fetcher:         fetcher,
		builder:         builder,
		clock:           clock,
	}
}

func (p *Processor) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
	close(ready)

	httpClient := cf_http.NewClient()
	httpClient.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: p.skipCertVerify,
			MinVersion:         tls.VersionTLS10,
		},
	}

	timer := p.clock.NewTimer(p.pollingInterval)
	stop := p.sync(signals, httpClient)

	for {
		if stop {
			return nil
		}

		select {
		case <-signals:
			return nil
		case <-timer.C():
			stop = p.sync(signals, httpClient)
			timer.Reset(p.pollingInterval)
		}
	}
}

func (p *Processor) sync(signals <-chan os.Signal, httpClient *http.Client) bool {
	start := p.clock.Now()
	defer func() {
		duration := p.clock.Now().Sub(start)
		syncDesiredLRPsDuration.Send(duration)
	}()

	logger := p.logger.Session("sync")

	existing, err := p.getDesiredLRPs(logger)
	if err != nil {
		return false
	}

	differ := NewDiffer(existing)

	cancel := make(chan struct{})

	fingerprints, fingerprintErrors := p.fetcher.FetchFingerprints(
		logger,
		cancel,
		httpClient,
	)

	diffErrors := differ.Diff(
		logger,
		cancel,
		fingerprints,
	)

	missingApps, missingAppsErrors := p.fetcher.FetchDesiredApps(
		logger,
		cancel,
		httpClient,
		differ.Missing(),
	)

	createErrors := p.createMissingDesiredLRPs(logger, cancel, missingApps)

	staleApps, staleAppErrors := p.fetcher.FetchDesiredApps(
		logger,
		cancel,
		httpClient,
		differ.Stale(),
	)

	updateErrors := p.updateStaleDesiredLRPs(logger, cancel, staleApps)

	bumpFreshness := true
	success := true

	fingerprintErrors, fingerprintErrorCount := countErrors(fingerprintErrors)

	errors := mergeErrors(
		fingerprintErrors,
		diffErrors,
		missingAppsErrors,
		staleAppErrors,
		createErrors,
		updateErrors,
	)

process_loop:
	for {
		select {
		case err, open := <-errors:
			if err != nil {
				logger.Error("not-bumping-freshness-because-of", err)
				bumpFreshness = false
			}
			if !open {
				break process_loop
			}
		case <-signals:
			close(cancel)
			return true
		}
	}

	if <-fingerprintErrorCount != 0 {
		logger.Error("failed-to-fetch-all-cc-fingerprints", nil)
		success = false
	}

	if success {
		deleteList := <-differ.Deleted()
		p.deleteExcess(logger, cancel, deleteList)
	}

	if bumpFreshness && success {
		logger.Info("bumping-freshness")

		err = p.receptorClient.UpsertDomain(recipebuilder.LRPDomain, p.domainTTL)
		if err != nil {
			logger.Error("failed-to-upsert-domain", err)
		}
	}

	return false
}

func (p *Processor) createMissingDesiredLRPs(
	logger lager.Logger,
	cancel <-chan struct{},
	missing <-chan []cc_messages.DesireAppRequestFromCC,
) <-chan error {
	logger = logger.Session("create-missing-desired-lrps")

	errc := make(chan error, 1)

	go func() {
		defer close(errc)

		for {
			var desireAppRequests []cc_messages.DesireAppRequestFromCC

			select {
			case <-cancel:
				return

			case selected, open := <-missing:
				if !open {
					return
				}

				desireAppRequests = selected
			}

			logger.Info("processing-batch", lager.Data{"size": len(desireAppRequests)})

			for _, desireAppRequest := range desireAppRequests {
				createReq, err := p.builder.Build(&desireAppRequest)
				if err != nil {
					logger.Error("failed-to-build-create-desired-lrp-request", err, lager.Data{
						"desire-app-request": desireAppRequest,
					})
					errc <- err
					continue
				}

				err = p.receptorClient.CreateDesiredLRP(*createReq)
				if err != nil {
					logger.Error("failed-to-create-desired-lrp", err, lager.Data{
						"create-request": createReq,
					})
					errc <- err
					continue
				}
			}
		}
	}()

	return errc
}

func (p *Processor) updateStaleDesiredLRPs(
	logger lager.Logger,
	cancel <-chan struct{},
	stale <-chan []cc_messages.DesireAppRequestFromCC,
) <-chan error {
	logger = logger.Session("update-stale-desired-lrps")

	errc := make(chan error, 1)

	go func() {
		defer close(errc)

		for {
			var staleAppRequests []cc_messages.DesireAppRequestFromCC

			select {
			case <-cancel:
				return

			case selected, open := <-stale:
				if !open {
					return
				}

				staleAppRequests = selected
			}

			logger.Info("processing-batch", lager.Data{"size": len(staleAppRequests)})

			for _, desireAppRequest := range staleAppRequests {
				updateReq := receptor.DesiredLRPUpdateRequest{}
				updateReq.Instances = &desireAppRequest.NumInstances
				updateReq.Annotation = &desireAppRequest.ETag
				updateReq.Routes = cc_messages.NewRoutingInfo(desireAppRequest.Hostnames, recipebuilder.DefaultPort)

				err := p.receptorClient.UpdateDesiredLRP(desireAppRequest.ProcessGuid, updateReq)
				if err != nil {
					logger.Error("failed-to-update-stale-lrp", err, lager.Data{
						"update-request": updateReq,
					})
					errc <- err
					continue
				}
			}
		}
	}()

	return errc
}

func (p *Processor) getDesiredLRPs(logger lager.Logger) ([]receptor.DesiredLRPResponse, error) {
	logger.Info("getting-desired-lrps-from-bbs")

	existing, err := p.receptorClient.DesiredLRPsByDomain(recipebuilder.LRPDomain)
	if err != nil {
		logger.Error("failed-to-get-desired-lrps", err)
		return nil, err
	}

	logger.Info("got-desired-lrps-from-bbs", lager.Data{"count": len(existing)})
	return existing, nil
}

func (p *Processor) deleteExcess(logger lager.Logger, cancel <-chan struct{}, excess []string) {
	logger = logger.Session("delete-excess")

	logger.Info(
		"processing-batch",
		lager.Data{"size": len(excess)},
	)

	for _, deleteGuid := range excess {
		err := p.receptorClient.DeleteDesiredLRP(deleteGuid)
		if err != nil {
			logger.Error(
				"failed-to-delete-desired-lrp",
				err,
				lager.Data{"delete-request": deleteGuid},
			)
		}
	}
}

func countErrors(source <-chan error) (<-chan error, <-chan int) {
	count := make(chan int, 1)
	dest := make(chan error, 1)
	var errorCount int

	wg := sync.WaitGroup{}
	wg.Add(1)

	go func() {
		for e := range source {
			errorCount++
			dest <- e
		}

		close(dest)
		wg.Done()
	}()

	go func() {
		wg.Wait()

		count <- errorCount
		close(count)
	}()

	return dest, count
}

func mergeErrors(channels ...<-chan error) <-chan error {
	out := make(chan error)
	wg := sync.WaitGroup{}

	for _, ch := range channels {
		wg.Add(1)

		go func(c <-chan error) {
			for e := range c {
				out <- e
			}
			wg.Done()
		}(ch)
	}

	go func() {
		wg.Wait()
		close(out)
	}()

	return out
}
<|endoftext|>


<commit_before>/**
 *  Copyright 2014 Paul Querna
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

package ffjsoninception

import (
	"bytes"
	"go/format"
	"text/template"
)

const ffjsonTemplate = `
// DO NOT EDIT!
// Code generated by ffjson <https://github.com/pquerna/ffjson>
// source: {{.InputPath}}
// DO NOT EDIT!

package {{.PackageName}}

import (
{{range $k, $v := .OutputImports}}{{$k}}
{{end}}
)

{{range .OutputFuncs}}
{{.}}
{{end}}

`

func RenderTemplate(ic *Inception) ([]byte, error) {
	t := template.Must(template.New("ffjson.go").Parse(ffjsonTemplate))
	buf := new(bytes.Buffer)
	err := t.Execute(buf, ic)
	if err != nil {
		return nil, err
	}
	return format.Source(buf.Bytes())
}

func tplStr(t *template.Template, data interface{}) string {
	buf := bytes.Buffer{}
	err := t.Execute(&buf, data)
	if err != nil {
		panic(err)
	}
	return buf.String()
}
<commit_msg>Adopt the standartized generated code header<commit_after>/**
 *  Copyright 2014 Paul Querna
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

package ffjsoninception

import (
	"bytes"
	"go/format"
	"text/template"
)

const ffjsonTemplate = `
// Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT.
// source: {{.InputPath}}

package {{.PackageName}}

import (
{{range $k, $v := .OutputImports}}{{$k}}
{{end}}
)

{{range .OutputFuncs}}
{{.}}
{{end}}

`

func RenderTemplate(ic *Inception) ([]byte, error) {
	t := template.Must(template.New("ffjson.go").Parse(ffjsonTemplate))
	buf := new(bytes.Buffer)
	err := t.Execute(buf, ic)
	if err != nil {
		return nil, err
	}
	return format.Source(buf.Bytes())
}

func tplStr(t *template.Template, data interface{}) string {
	buf := bytes.Buffer{}
	err := t.Execute(&buf, data)
	if err != nil {
		panic(err)
	}
	return buf.String()
}
<|endoftext|>


<commit_before>// The quantile package implements the algorithm in the paper Effective
// Computation of Biased Quantiles over Data Streams with both invarients.
//
// This package is useful for calculating hight-biased and targeted quantiles
// for large datasets within low memory and CPU bounds. You trade a small
// amount of accuracy in rank selection for efficiency.
//
// Multiple Stream's can be merged before a Query, allowing clients to be
// distributed across threads. See Stream.Merge and Stream.Samples.
//
// For more detailed information about the algorithm, see:
// http://www.cs.rutgers.edu/~muthu/bquant.pdf
package quantile

import (
	"container/list"
	"math"
	"sort"
)

// Sample holds an observed value and meta information for compression. JSON
// tags have been added for convenience.
type Sample struct {
	Value float64 `json:",string"`
	Width float64 `json:",string"`
	Delta float64 `json:",string"`
}

// Samples represents a slice of samples. It implements sort.Interface.
type Samples []Sample

func (a Samples) Len() int {
	return len(a)
}

func (a Samples) Less(i, j int) bool {
	return a[i].Value < a[j].Value
}

func (a Samples) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

type Invariant func(s *stream, r float64) float64

// Biased returns an Invariant for high-biased (>50th) quantiles not known a
// priori with associated error bounds e (usually 0.01).
// Biased requires space bounds O(1/e * log(en)), where n is the total inserts, in the worst case.
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
func Biased(e float64) Invariant {
	return func(s *stream, r float64) float64 {
		return 2 * e * r
	}
}

// Targeted returns an Invariant that is only concerned with a set
// of quantile values with associated error bounds e (usually 0.01) that are supplied a priori.
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
func Targeted(e float64, quantiles ...float64) Invariant {
	return func(s *stream, r float64) float64 {
		var m float64 = math.MaxFloat64
		var f float64
		for _, q := range quantiles {
			if q*s.n <= r {
				f = (2 * e * r) / q
			} else {
				f = (2 * e * (s.n - r)) / (1 - q)
			}
			m = math.Min(m, f)
		}
		return m
	}
}

// Stream calculates quantiles for a stream of float64s.
type Stream struct {
	*stream
	b Samples
}

// New returns an initialized Stream with the Invariant ƒ.
func New(ƒ Invariant) *Stream {
	x := &stream{ƒ: ƒ, l: list.New()}
	return &Stream{x, make(Samples, 0, 500)}
}

// Insert inserts v into the stream.
func (s *Stream) Insert(v float64) {
	s.insert(Sample{Value: v, Width: 1})
}

func (s *Stream) insert(sample Sample) {
	s.b = append(s.b, sample)
	if len(s.b) == cap(s.b) {
		s.flush()
		s.compress()
	}
}

// Query returns the calculated qth percentiles value. If q is not in the set
// of quantiles provided to New, Query will have non-deterministic results.
func (s *Stream) Query(q float64) float64 {
	if s.flushed() {
		// Fast path when there hasn't been enough data for a flush;
		// this also yeilds better accuracy for small sets of data.
		i := float64(len(s.b)) * q
		return s.b[int(i)].Value
	}
	s.flush()
	return s.stream.query(q)
}

// Merge merges samples into the underlying streams samples. This is handy when
// merging multiple streams from separate threads.
func (s *Stream) Merge(samples Samples) {
	s.stream.merge(samples)
}

// Reset reinitializes and clears the list reusing the samples buffer memory.
func (s *Stream) Reset() {
	s.stream.reset()
	s.b = s.b[:0]
}

// Samples returns stream samples held by s.
func (s *Stream) Samples() Samples {
	if !s.flushed() {
		return s.b
	}
	return s.stream.samples()
}

func (s *Stream) flush() {
	sort.Sort(s.b)
	s.stream.merge(s.b)
	s.b = s.b[:0]
}

func (s *Stream) flushed() bool {
	return s.stream.l.Len() == 0
}

type stream struct {
	n float64
	l *list.List
	ƒ Invariant
}

func (s *stream) reset() {
	s.l.Init()
	s.n = 0
}

func (s *stream) insert(v float64) {
	fn := s.mergeFunc()
	fn(v, 1)
}

func (s *stream) merge(samples Samples) {
	fn := s.mergeFunc()
	for _, s := range samples {
		fn(s.Value, s.Width)
	}
}

func (s *stream) mergeFunc() func(v, w float64) {
	// NOTE: I used a goto over defer because it bought me a few extra
	// nanoseconds. I know. I know.
	var r float64
	e := s.l.Front()
	return func(v, w float64) {
		for ; e != nil; e = e.Next() {
			c := e.Value.(*Sample)
			if c.Value > v {
				sm := &Sample{v, w, math.Floor(s.ƒ(s, r)) - 1}
				s.l.InsertBefore(sm, e)
				goto inserted
			}
			r += c.Width
		}
		s.l.PushBack(&Sample{v, w, 0})
	inserted:
		s.n += w
	}
}

// Count returns the total number of samples observed in the stream
// since initialization.
func (s *stream) Count() int {
	return int(s.n)
}

func (s *stream) query(q float64) float64 {
	e := s.l.Front()
	t := math.Ceil(q * s.n)
	t += math.Ceil(s.ƒ(s, t) / 2)
	p := e.Value.(*Sample)
	e = e.Next()
	r := float64(0)
	for e != nil {
		c := e.Value.(*Sample)
		if r+c.Width+c.Delta > t {
			return p.Value
		}
		r += p.Width
		p = c
		e = e.Next()
	}
	return p.Value
}

func (s *stream) compress() {
	if s.l.Len() < 2 {
		return
	}
	e := s.l.Back()
	x := e.Value.(*Sample)
	r := s.n - 1 - x.Width
	e = e.Prev()
	for e != nil {
		c := e.Value.(*Sample)
		if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
			x.Width += c.Width
			o := e
			e = e.Prev()
			s.l.Remove(o)
		} else {
			x = c
			e = e.Prev()
		}
		r -= c.Width
	}
}

func (s *stream) samples() Samples {
	samples := make(Samples, 0, s.l.Len())
	for e := s.l.Front(); e != nil; e = e.Next() {
		samples = append(samples, *e.Value.(*Sample))
	}
	return samples
}
<commit_msg>defer time/space/error bounds to paper<commit_after>// The quantile package implements the algorithm in the paper Effective
// Computation of Biased Quantiles over Data Streams with both invarients.
//
// This package is useful for calculating hight-biased and targeted quantiles
// for large datasets within low memory and CPU bounds. You trade a small
// amount of accuracy in rank selection for efficiency.
//
// Multiple Stream's can be merged before a Query, allowing clients to be
// distributed across threads. See Stream.Merge and Stream.Samples.
//
// For more detailed information about the algorithm, see:
// http://www.cs.rutgers.edu/~muthu/bquant.pdf
package quantile

import (
	"container/list"
	"math"
	"sort"
)

// Sample holds an observed value and meta information for compression. JSON
// tags have been added for convenience.
type Sample struct {
	Value float64 `json:",string"`
	Width float64 `json:",string"`
	Delta float64 `json:",string"`
}

// Samples represents a slice of samples. It implements sort.Interface.
type Samples []Sample

func (a Samples) Len() int {
	return len(a)
}

func (a Samples) Less(i, j int) bool {
	return a[i].Value < a[j].Value
}

func (a Samples) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

type Invariant func(s *stream, r float64) float64

// Biased returns an Invariant for high-biased (>50th) quantiles not known a
// priori with associated error bounds e (usually 0.01).
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
func Biased(e float64) Invariant {
	return func(s *stream, r float64) float64 {
		return 2 * e * r
	}
}

// Targeted returns an Invariant that is only concerned with a set
// of quantile values with associated error bounds e (usually 0.01) that are supplied a priori.
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
func Targeted(e float64, quantiles ...float64) Invariant {
	return func(s *stream, r float64) float64 {
		var m float64 = math.MaxFloat64
		var f float64
		for _, q := range quantiles {
			if q*s.n <= r {
				f = (2 * e * r) / q
			} else {
				f = (2 * e * (s.n - r)) / (1 - q)
			}
			m = math.Min(m, f)
		}
		return m
	}
}

// Stream calculates quantiles for a stream of float64s.
type Stream struct {
	*stream
	b Samples
}

// New returns an initialized Stream with the Invariant ƒ.
func New(ƒ Invariant) *Stream {
	x := &stream{ƒ: ƒ, l: list.New()}
	return &Stream{x, make(Samples, 0, 500)}
}

// Insert inserts v into the stream.
func (s *Stream) Insert(v float64) {
	s.insert(Sample{Value: v, Width: 1})
}

func (s *Stream) insert(sample Sample) {
	s.b = append(s.b, sample)
	if len(s.b) == cap(s.b) {
		s.flush()
		s.compress()
	}
}

// Query returns the calculated qth percentiles value. If q is not in the set
// of quantiles provided to New, Query will have non-deterministic results.
func (s *Stream) Query(q float64) float64 {
	if s.flushed() {
		// Fast path when there hasn't been enough data for a flush;
		// this also yeilds better accuracy for small sets of data.
		i := float64(len(s.b)) * q
		return s.b[int(i)].Value
	}
	s.flush()
	return s.stream.query(q)
}

// Merge merges samples into the underlying streams samples. This is handy when
// merging multiple streams from separate threads.
func (s *Stream) Merge(samples Samples) {
	s.stream.merge(samples)
}

// Reset reinitializes and clears the list reusing the samples buffer memory.
func (s *Stream) Reset() {
	s.stream.reset()
	s.b = s.b[:0]
}

// Samples returns stream samples held by s.
func (s *Stream) Samples() Samples {
	if !s.flushed() {
		return s.b
	}
	return s.stream.samples()
}

func (s *Stream) flush() {
	sort.Sort(s.b)
	s.stream.merge(s.b)
	s.b = s.b[:0]
}

func (s *Stream) flushed() bool {
	return s.stream.l.Len() == 0
}

type stream struct {
	n float64
	l *list.List
	ƒ Invariant
}

func (s *stream) reset() {
	s.l.Init()
	s.n = 0
}

func (s *stream) insert(v float64) {
	fn := s.mergeFunc()
	fn(v, 1)
}

func (s *stream) merge(samples Samples) {
	fn := s.mergeFunc()
	for _, s := range samples {
		fn(s.Value, s.Width)
	}
}

func (s *stream) mergeFunc() func(v, w float64) {
	// NOTE: I used a goto over defer because it bought me a few extra
	// nanoseconds. I know. I know.
	var r float64
	e := s.l.Front()
	return func(v, w float64) {
		for ; e != nil; e = e.Next() {
			c := e.Value.(*Sample)
			if c.Value > v {
				sm := &Sample{v, w, math.Floor(s.ƒ(s, r)) - 1}
				s.l.InsertBefore(sm, e)
				goto inserted
			}
			r += c.Width
		}
		s.l.PushBack(&Sample{v, w, 0})
	inserted:
		s.n += w
	}
}

// Count returns the total number of samples observed in the stream
// since initialization.
func (s *stream) Count() int {
	return int(s.n)
}

func (s *stream) query(q float64) float64 {
	e := s.l.Front()
	t := math.Ceil(q * s.n)
	t += math.Ceil(s.ƒ(s, t) / 2)
	p := e.Value.(*Sample)
	e = e.Next()
	r := float64(0)
	for e != nil {
		c := e.Value.(*Sample)
		if r+c.Width+c.Delta > t {
			return p.Value
		}
		r += p.Width
		p = c
		e = e.Next()
	}
	return p.Value
}

func (s *stream) compress() {
	if s.l.Len() < 2 {
		return
	}
	e := s.l.Back()
	x := e.Value.(*Sample)
	r := s.n - 1 - x.Width
	e = e.Prev()
	for e != nil {
		c := e.Value.(*Sample)
		if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
			x.Width += c.Width
			o := e
			e = e.Prev()
			s.l.Remove(o)
		} else {
			x = c
			e = e.Prev()
		}
		r -= c.Width
	}
}

func (s *stream) samples() Samples {
	samples := make(Samples, 0, s.l.Len())
	for e := s.l.Front(); e != nil; e = e.Next() {
		samples = append(samples, *e.Value.(*Sample))
	}
	return samples
}
<|endoftext|>


<commit_before>package gorm

import (
	"fmt"
	"reflect"
)

func Query(scope *Scope) {
	defer scope.Trace(NowFunc())

	var (
		isSlice        bool
		isPtr          bool
		anyRecordFound bool
		destType       reflect.Type
	)

	var dest = scope.IndirectValue()
	if value, ok := scope.InstanceGet("gorm:query_destination"); ok {
		dest = reflect.Indirect(reflect.ValueOf(value))
	}

	if orderBy, ok := scope.InstanceGet("gorm:order_by_primary_key"); ok {
		if primaryKey := scope.PrimaryKey(); primaryKey != "" {
			scope.Search = scope.Search.clone().order(fmt.Sprintf("%v.%v %v", scope.QuotedTableName(), primaryKey, orderBy))
		}
	}

	if dest.Kind() == reflect.Slice {
		isSlice = true
		destType = dest.Type().Elem()
		if destType.Kind() == reflect.Ptr {
			isPtr = true
			destType = destType.Elem()
		}
	} else {
		scope.Search = scope.Search.clone().limit(1)
	}

	scope.prepareQuerySql()

	if !scope.HasError() {
		rows, err := scope.DB().Query(scope.Sql, scope.SqlVars...)

		if scope.Err(err) != nil {
			return
		}

		defer rows.Close()
		columns, _ := rows.Columns()
		for rows.Next() {
			anyRecordFound = true
			elem := dest
			if isSlice {
				elem = reflect.New(destType).Elem()
			}

			var values = make([]interface{}, len(columns))

			fields := scope.New(elem.Addr().Interface()).Fields()
			for index, column := range columns {
				if field, ok := fields[column]; ok {
					if field.Field.Kind() == reflect.Ptr {
						values[index] = field.Field.Addr().Interface()
					} else {
						values[index] = reflect.New(reflect.PtrTo(field.Field.Type())).Interface()
					}
				} else {
					var value interface{}
					values[index] = &value
				}
			}

			scope.Err(rows.Scan(values...))

			for index, column := range columns {
				value := values[index]
				if field, ok := fields[column]; ok {
					if field.Field.Kind() == reflect.Ptr {
						field.Field.Set(reflect.ValueOf(value).Elem())
					} else if v := reflect.ValueOf(value).Elem().Elem(); v.IsValid() {
						field.Field.Set(v)
					}
				}
			}

			if isSlice {
				if isPtr {
					dest.Set(reflect.Append(dest, elem.Addr()))
				} else {
					dest.Set(reflect.Append(dest, elem))
				}
			}
		}

		if !anyRecordFound && !isSlice {
			scope.Err(RecordNotFound)
		}
	}
}

func AfterQuery(scope *Scope) {
	scope.CallMethod("AfterFind")
}

func init() {
	DefaultCallback.Query().Register("gorm:query", Query)
	DefaultCallback.Query().Register("gorm:after_query", AfterQuery)
}
<commit_msg>Add RowsAffected for query<commit_after>package gorm

import (
	"fmt"
	"reflect"
)

func Query(scope *Scope) {
	defer scope.Trace(NowFunc())

	var (
		isSlice        bool
		isPtr          bool
		anyRecordFound bool
		destType       reflect.Type
	)

	var dest = scope.IndirectValue()
	if value, ok := scope.InstanceGet("gorm:query_destination"); ok {
		dest = reflect.Indirect(reflect.ValueOf(value))
	}

	if orderBy, ok := scope.InstanceGet("gorm:order_by_primary_key"); ok {
		if primaryKey := scope.PrimaryKey(); primaryKey != "" {
			scope.Search = scope.Search.clone().order(fmt.Sprintf("%v.%v %v", scope.QuotedTableName(), primaryKey, orderBy))
		}
	}

	if dest.Kind() == reflect.Slice {
		isSlice = true
		destType = dest.Type().Elem()
		if destType.Kind() == reflect.Ptr {
			isPtr = true
			destType = destType.Elem()
		}
	} else {
		scope.Search = scope.Search.clone().limit(1)
	}

	scope.prepareQuerySql()

	if !scope.HasError() {
		rows, err := scope.DB().Query(scope.Sql, scope.SqlVars...)
		scope.db.RowsAffected = 0

		if scope.Err(err) != nil {
			return
		}

		columns, _ := rows.Columns()

		defer rows.Close()
		for rows.Next() {
			scope.db.RowsAffected += 1

			anyRecordFound = true
			elem := dest
			if isSlice {
				elem = reflect.New(destType).Elem()
			}

			var values = make([]interface{}, len(columns))

			fields := scope.New(elem.Addr().Interface()).Fields()
			for index, column := range columns {
				if field, ok := fields[column]; ok {
					if field.Field.Kind() == reflect.Ptr {
						values[index] = field.Field.Addr().Interface()
					} else {
						values[index] = reflect.New(reflect.PtrTo(field.Field.Type())).Interface()
					}
				} else {
					var value interface{}
					values[index] = &value
				}
			}

			scope.Err(rows.Scan(values...))

			for index, column := range columns {
				value := values[index]
				if field, ok := fields[column]; ok {
					if field.Field.Kind() == reflect.Ptr {
						field.Field.Set(reflect.ValueOf(value).Elem())
					} else if v := reflect.ValueOf(value).Elem().Elem(); v.IsValid() {
						field.Field.Set(v)
					}
				}
			}

			if isSlice {
				if isPtr {
					dest.Set(reflect.Append(dest, elem.Addr()))
				} else {
					dest.Set(reflect.Append(dest, elem))
				}
			}
		}

		if !anyRecordFound && !isSlice {
			scope.Err(RecordNotFound)
		}
	}
}

func AfterQuery(scope *Scope) {
	scope.CallMethod("AfterFind")
}

func init() {
	DefaultCallback.Query().Register("gorm:query", Query)
	DefaultCallback.Query().Register("gorm:after_query", AfterQuery)
}
<|endoftext|>


<commit_before>package lazyquicktime

import "fmt"
import "image"
import "errors"

import "time"
import "log"

import "github.com/amarburg/go-lazyfs"
import "github.com/amarburg/go-quicktime"
import "github.com/amarburg/go-prores-ffmpeg"

type LazyQuicktime struct {
	file lazyfs.FileSource
	Tree quicktime.AtomArray
	Trak quicktime.TRAKAtom
	Stbl *quicktime.STBLAtom
	Mvhd quicktime.MVHDAtom

	FileSize int64
}

func LoadMovMetadata(file lazyfs.FileSource) (*LazyQuicktime, error) {

	mov := &LazyQuicktime{file: file}

	sz, err := file.FileSize()
	if sz < 0 || err != nil {
		return mov, fmt.Errorf("Unable to retrieve file size.")
	}

	mov.FileSize = sz

	set_eagerload := func(conf *quicktime.BuildTreeConfig) {
		conf.EagerloadTypes = []string{"moov"}
	}

	fmt.Println("Reading Mov of size ", mov.FileSize)
	tree, err := quicktime.BuildTree(file, uint64(mov.FileSize), set_eagerload)

	if err != nil {
		return mov, err
	}
	mov.Tree = tree

	moov := mov.Tree.FindAtom("moov")
	if moov == nil {
		return mov, errors.New("Can't find MOOV atom")
	}

	mvhd := moov.FindAtom("mvhd")
	if mvhd == nil {
		return mov, errors.New("Couldn't find MVHD in the moov atom")
	}
	mov.Mvhd, _ = quicktime.ParseMVHD(mvhd)

	tracks := moov.FindAtoms("trak")
	if tracks == nil || len(tracks) == 0 {
		return mov, errors.New("Couldn't find any TRAKs in the MOOV")
	}

	var track *quicktime.Atom = nil
	for i, t := range tracks {
		mdia := t.FindAtom("mdia")
		if mdia == nil {
			fmt.Println("No mdia track", i)
			continue
		}

		minf := mdia.FindAtom("minf")
		if minf == nil {
			fmt.Println("No minf track", i)
			continue
		}

		if minf.FindAtom("vmhd") != nil {
			track = t
			break
		}
	}

	if track == nil {
		return mov, errors.New("Couldn't identify the Video track")
	}

	mov.Trak, err = quicktime.ParseTRAK(track)
	if err != nil {
		return mov, errors.New(fmt.Sprintf("Unable to parse TRAK atom: %s", err.Error()))
	}

	mov.Stbl = &mov.Trak.Mdia.Minf.Stbl // Just an alias

	return mov, nil
}

func (mov *LazyQuicktime) NumFrames() int {
	return mov.Stbl.NumFrames()
}

func (mov *LazyQuicktime) Duration() float32 {
	return mov.Mvhd.Duration()
}

func (mov *LazyQuicktime) ExtractFrame(frame int) (image.Image, error) {

	frame_offset, frame_size, _ := mov.Stbl.SampleOffsetSize(frame)

	//fmt.Printf("Extracting frame %d at offset %d size %d\n", frame, frame_offset, frame_size)

	buf := make([]byte, frame_size)

	if buf == nil {
		return nil, fmt.Errorf("Couldn't make buffer of size %d", frame_size)
	}

	startRead := time.Now()
	n, _ := mov.file.ReadAt(buf, frame_offset)
	log.Printf("HTTP read took %s", time.Since(startRead))

	if n != frame_size {
		return nil, fmt.Errorf("Tried to read %d bytes but got %d instead", frame_size, n)
	}

	width, height := int(mov.Trak.Tkhd.Width), int(mov.Trak.Tkhd.Height)

	startDecode := time.Now()
	img, err := prores.DecodeProRes(buf, width, height)
	log.Printf("Prores decode took %s", time.Since(startDecode))

	return img, err

}
<commit_msg>Removed more debugging output.<commit_after>package lazyquicktime

import "fmt"
import "image"
import "errors"

import "time"
import "log"

import "github.com/amarburg/go-lazyfs"
import "github.com/amarburg/go-quicktime"
import "github.com/amarburg/go-prores-ffmpeg"

type LazyQuicktime struct {
	file lazyfs.FileSource
	Tree quicktime.AtomArray
	Trak quicktime.TRAKAtom
	Stbl *quicktime.STBLAtom
	Mvhd quicktime.MVHDAtom

	FileSize int64
}

func LoadMovMetadata(file lazyfs.FileSource) (*LazyQuicktime, error) {

	mov := &LazyQuicktime{file: file}

	sz, err := file.FileSize()
	if sz < 0 || err != nil {
		return mov, fmt.Errorf("Unable to retrieve file size.")
	}

	mov.FileSize = sz

	set_eagerload := func(conf *quicktime.BuildTreeConfig) {
		conf.EagerloadTypes = []string{"moov"}
	}

	//fmt.Println("Reading Mov of size ", mov.FileSize)
	tree, err := quicktime.BuildTree(file, uint64(mov.FileSize), set_eagerload)

	if err != nil {
		return mov, err
	}
	mov.Tree = tree

	moov := mov.Tree.FindAtom("moov")
	if moov == nil {
		return mov, errors.New("Can't find MOOV atom")
	}

	mvhd := moov.FindAtom("mvhd")
	if mvhd == nil {
		return mov, errors.New("Couldn't find MVHD in the moov atom")
	}
	mov.Mvhd, _ = quicktime.ParseMVHD(mvhd)

	tracks := moov.FindAtoms("trak")
	if tracks == nil || len(tracks) == 0 {
		return mov, errors.New("Couldn't find any TRAKs in the MOOV")
	}

	var track *quicktime.Atom = nil
	for i, t := range tracks {
		mdia := t.FindAtom("mdia")
		if mdia == nil {
			fmt.Println("No mdia track", i)
			continue
		}

		minf := mdia.FindAtom("minf")
		if minf == nil {
			fmt.Println("No minf track", i)
			continue
		}

		if minf.FindAtom("vmhd") != nil {
			track = t
			break
		}
	}

	if track == nil {
		return mov, errors.New("Couldn't identify the Video track")
	}

	mov.Trak, err = quicktime.ParseTRAK(track)
	if err != nil {
		return mov, errors.New(fmt.Sprintf("Unable to parse TRAK atom: %s", err.Error()))
	}

	mov.Stbl = &mov.Trak.Mdia.Minf.Stbl // Just an alias

	return mov, nil
}

func (mov *LazyQuicktime) NumFrames() int {
	return mov.Stbl.NumFrames()
}

func (mov *LazyQuicktime) Duration() float32 {
	return mov.Mvhd.Duration()
}

func (mov *LazyQuicktime) ExtractFrame(frame int) (image.Image, error) {

	frame_offset, frame_size, _ := mov.Stbl.SampleOffsetSize(frame)

	//fmt.Printf("Extracting frame %d at offset %d size %d\n", frame, frame_offset, frame_size)

	buf := make([]byte, frame_size)

	if buf == nil {
		return nil, fmt.Errorf("Couldn't make buffer of size %d", frame_size)
	}

	startRead := time.Now()
	n, _ := mov.file.ReadAt(buf, frame_offset)
	log.Printf("HTTP read took %s", time.Since(startRead))

	if n != frame_size {
		return nil, fmt.Errorf("Tried to read %d bytes but got %d instead", frame_size, n)
	}

	width, height := int(mov.Trak.Tkhd.Width), int(mov.Trak.Tkhd.Height)

	startDecode := time.Now()
	img, err := prores.DecodeProRes(buf, width, height)
	log.Printf("Prores decode took %s", time.Since(startDecode))

	return img, err

}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	ggio "github.com/gogo/protobuf/io"
	p2p_host "github.com/libp2p/go-libp2p-host"
	p2p_net "github.com/libp2p/go-libp2p-net"
	p2p_peer "github.com/libp2p/go-libp2p-peer"
	p2p_pstore "github.com/libp2p/go-libp2p-peerstore"
	mc "github.com/mediachain/concat/mc"
	pb "github.com/mediachain/concat/proto"
	homedir "github.com/mitchellh/go-homedir"
	"log"
	"os"
	"sync"
)

type Directory struct {
	mc.PeerIdentity
	host  p2p_host.Host
	peers map[p2p_peer.ID]p2p_pstore.PeerInfo
	mx    sync.Mutex
}

func (dir *Directory) registerHandler(s p2p_net.Stream) {
	defer s.Close()

	pid := s.Conn().RemotePeer()
	paddr := s.Conn().RemoteMultiaddr()
	log.Printf("directory/register: new stream from %s at %s", pid.Pretty(), paddr.String())

	var req pb.RegisterPeer
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)

	for {
		err := r.ReadMsg(&req)
		if err != nil {
			break
		}

		if req.Info == nil {
			log.Printf("directory/register: empty peer info from %s", pid.Pretty())
			break
		}

		pinfo, err := mc.PBToPeerInfo(req.Info)
		if err != nil {
			log.Printf("directory/register: bad peer info from %s", pid.Pretty())
			break
		}

		if pinfo.ID != pid {
			log.Printf("directory/register: bogus peer info from %s", pid.Pretty())
			break
		}

		dir.registerPeer(pinfo)

		req.Reset()
	}

	dir.unregisterPeer(pid)
}

func (dir *Directory) lookupHandler(s p2p_net.Stream) {
	defer s.Close()

	pid := s.Conn().RemotePeer()
	paddr := s.Conn().RemoteMultiaddr()
	log.Printf("directory/lookup: new stream from %s at %s", pid.Pretty(), paddr.String())

	var req pb.LookupPeerRequest
	var res pb.LookupPeerResponse

	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	w := ggio.NewDelimitedWriter(s)

	for {
		err := r.ReadMsg(&req)
		if err != nil {
			break
		}

		xid, err := p2p_peer.IDB58Decode(req.Id)
		if err != nil {
			log.Printf("directory/lookup: bad request from %s", pid.Pretty())
			break
		}

		pinfo, ok := dir.lookupPeer(xid)
		if ok {
			var pbpi pb.PeerInfo
			mc.PBFromPeerInfo(&pbpi, pinfo)
			res.Peer = &pbpi
		}

		err = w.WriteMsg(&res)
		if err != nil {
			break
		}

		req.Reset()
		res.Reset()
	}
}

func (dir *Directory) listHandler(s p2p_net.Stream) {
	defer s.Close()

	pid := s.Conn().RemotePeer()
	paddr := s.Conn().RemoteMultiaddr()
	log.Printf("directory/list: new stream from %s at %s", pid.Pretty(), paddr.String())

	var req pb.ListPeersRequest
	var res pb.ListPeersResponse

	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	w := ggio.NewDelimitedWriter(s)

	for {
		err := r.ReadMsg(&req)
		if err != nil {
			break
		}

		res.Peers = dir.listPeers()

		err = w.WriteMsg(&res)
		if err != nil {
			break
		}

		res.Reset()
	}
}

func (dir *Directory) registerPeer(info p2p_pstore.PeerInfo) {
	log.Printf("directory: register %s", info.ID.Pretty())
	dir.mx.Lock()
	dir.peers[info.ID] = info
	dir.mx.Unlock()
}

func (dir *Directory) unregisterPeer(pid p2p_peer.ID) {
	log.Printf("directory: unregister %s", pid.Pretty())
	dir.mx.Lock()
	delete(dir.peers, pid)
	dir.mx.Unlock()
}

func (dir *Directory) lookupPeer(pid p2p_peer.ID) (p2p_pstore.PeerInfo, bool) {
	dir.mx.Lock()
	pinfo, ok := dir.peers[pid]
	dir.mx.Unlock()
	return pinfo, ok
}

func (dir *Directory) listPeers() []string {
	dir.mx.Lock()
	lst := make([]string, 0, len(dir.peers))
	for pid, _ := range dir.peers {
		lst = append(lst, pid.Pretty())
	}
	dir.mx.Unlock()
	return lst
}

func main() {
	port := flag.Int("l", 9000, "Listen port")
	hdir := flag.String("d", "~/.mediachain/mcdir", "Directory home")
	flag.Parse()

	if len(flag.Args()) != 0 {
		fmt.Fprintf(os.Stderr, "Usage: %s [options ...]\nOptions:\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	home, err := homedir.Expand(*hdir)
	if err != nil {
		log.Fatal(err)
	}

	err = os.MkdirAll(home, 0755)
	if err != nil {
		log.Fatal(err)
	}

	id, err := mc.MakePeerIdentity(home)
	if err != nil {
		log.Fatal(err)
	}

	addr, err := mc.ParseAddress(fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", *port))
	if err != nil {
		log.Fatal(err)
	}

	host, err := mc.NewHost(context.Background(), id, addr)
	if err != nil {
		log.Fatal(err)
	}

	dir := &Directory{PeerIdentity: id, host: host, peers: make(map[p2p_peer.ID]p2p_pstore.PeerInfo)}
	host.SetStreamHandler("/mediachain/dir/register", dir.registerHandler)
	host.SetStreamHandler("/mediachain/dir/lookup", dir.lookupHandler)
	host.SetStreamHandler("/mediachain/dir/list", dir.listHandler)

	for _, addr := range host.Addrs() {
		if !mc.IsLinkLocalAddr(addr) {
			log.Printf("I am %s/%s", addr, id.Pretty())
		}
	}
	select {}
}
<commit_msg>mcdir: track publisher info<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	ggio "github.com/gogo/protobuf/io"
	p2p_host "github.com/libp2p/go-libp2p-host"
	p2p_net "github.com/libp2p/go-libp2p-net"
	p2p_peer "github.com/libp2p/go-libp2p-peer"
	p2p_pstore "github.com/libp2p/go-libp2p-peerstore"
	mc "github.com/mediachain/concat/mc"
	pb "github.com/mediachain/concat/proto"
	homedir "github.com/mitchellh/go-homedir"
	"log"
	"os"
	"sync"
)

type Directory struct {
	mc.PeerIdentity
	host  p2p_host.Host
	peers map[p2p_peer.ID]PeerRecord
	mx    sync.Mutex
}

type PeerRecord struct {
	peer      p2p_pstore.PeerInfo
	publisher *pb.PublisherInfo
}

func (dir *Directory) registerHandler(s p2p_net.Stream) {
	defer s.Close()

	pid := s.Conn().RemotePeer()
	paddr := s.Conn().RemoteMultiaddr()
	log.Printf("directory/register: new stream from %s at %s", pid.Pretty(), paddr.String())

	var req pb.RegisterPeer
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)

	for {
		err := r.ReadMsg(&req)
		if err != nil {
			break
		}

		if req.Info == nil {
			log.Printf("directory/register: empty peer info from %s", pid.Pretty())
			break
		}

		pinfo, err := mc.PBToPeerInfo(req.Info)
		if err != nil {
			log.Printf("directory/register: bad peer info from %s", pid.Pretty())
			break
		}

		if pinfo.ID != pid {
			log.Printf("directory/register: bogus peer info from %s", pid.Pretty())
			break
		}

		dir.registerPeer(PeerRecord{pinfo, req.Publisher})

		req.Reset()
	}

	dir.unregisterPeer(pid)
}

func (dir *Directory) lookupHandler(s p2p_net.Stream) {
	defer s.Close()

	pid := s.Conn().RemotePeer()
	paddr := s.Conn().RemoteMultiaddr()
	log.Printf("directory/lookup: new stream from %s at %s", pid.Pretty(), paddr.String())

	var req pb.LookupPeerRequest
	var res pb.LookupPeerResponse

	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	w := ggio.NewDelimitedWriter(s)

	for {
		err := r.ReadMsg(&req)
		if err != nil {
			break
		}

		xid, err := p2p_peer.IDB58Decode(req.Id)
		if err != nil {
			log.Printf("directory/lookup: bad request from %s", pid.Pretty())
			break
		}

		pinfo, ok := dir.lookupPeer(xid)
		if ok {
			var pbpi pb.PeerInfo
			mc.PBFromPeerInfo(&pbpi, pinfo)
			res.Peer = &pbpi
		}

		err = w.WriteMsg(&res)
		if err != nil {
			break
		}

		req.Reset()
		res.Reset()
	}
}

func (dir *Directory) listHandler(s p2p_net.Stream) {
	defer s.Close()

	pid := s.Conn().RemotePeer()
	paddr := s.Conn().RemoteMultiaddr()
	log.Printf("directory/list: new stream from %s at %s", pid.Pretty(), paddr.String())

	var req pb.ListPeersRequest
	var res pb.ListPeersResponse

	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	w := ggio.NewDelimitedWriter(s)

	for {
		err := r.ReadMsg(&req)
		if err != nil {
			break
		}

		res.Peers = dir.listPeers()

		err = w.WriteMsg(&res)
		if err != nil {
			break
		}

		res.Reset()
	}
}

func (dir *Directory) registerPeer(rec PeerRecord) {
	log.Printf("directory: register %s", rec.peer.ID.Pretty())
	dir.mx.Lock()
	dir.peers[rec.peer.ID] = rec
	dir.mx.Unlock()
}

func (dir *Directory) unregisterPeer(pid p2p_peer.ID) {
	log.Printf("directory: unregister %s", pid.Pretty())
	dir.mx.Lock()
	delete(dir.peers, pid)
	dir.mx.Unlock()
}

func (dir *Directory) lookupPeer(pid p2p_peer.ID) (p2p_pstore.PeerInfo, bool) {
	log.Printf("directory: lookup %s", pid.Pretty())
	dir.mx.Lock()
	rec, ok := dir.peers[pid]
	dir.mx.Unlock()
	return rec.peer, ok
}

func (dir *Directory) listPeers() []string {
	dir.mx.Lock()
	lst := make([]string, 0, len(dir.peers))
	for pid, _ := range dir.peers {
		lst = append(lst, pid.Pretty())
	}
	dir.mx.Unlock()
	return lst
}

func main() {
	port := flag.Int("l", 9000, "Listen port")
	hdir := flag.String("d", "~/.mediachain/mcdir", "Directory home")
	flag.Parse()

	if len(flag.Args()) != 0 {
		fmt.Fprintf(os.Stderr, "Usage: %s [options ...]\nOptions:\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	home, err := homedir.Expand(*hdir)
	if err != nil {
		log.Fatal(err)
	}

	err = os.MkdirAll(home, 0755)
	if err != nil {
		log.Fatal(err)
	}

	id, err := mc.MakePeerIdentity(home)
	if err != nil {
		log.Fatal(err)
	}

	addr, err := mc.ParseAddress(fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", *port))
	if err != nil {
		log.Fatal(err)
	}

	host, err := mc.NewHost(context.Background(), id, addr)
	if err != nil {
		log.Fatal(err)
	}

	dir := &Directory{PeerIdentity: id, host: host, peers: make(map[p2p_peer.ID]PeerRecord)}
	host.SetStreamHandler("/mediachain/dir/register", dir.registerHandler)
	host.SetStreamHandler("/mediachain/dir/lookup", dir.lookupHandler)
	host.SetStreamHandler("/mediachain/dir/list", dir.listHandler)

	for _, addr := range host.Addrs() {
		if !mc.IsLinkLocalAddr(addr) {
			log.Printf("I am %s/%s", addr, id.Pretty())
		}
	}
	select {}
}
<|endoftext|>


<commit_before>package torznab

import (
	"crypto/tls"
	"encoding/xml"
	//"errors"
	"flemzerd/indexers"
	log "flemzerd/logging"
	//"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
)

type TorznabIndexer struct {
	Name   string
	Url    string
	ApiKey string
}

type TorrentSearchResults struct {
	Torrents []TorznabTorrent `xml:"channel>item"`
}

type TorznabTorrent struct {
	Title       string `xml:"title"`
	Description string `xml:"description"`
	Guid        string `xml:"guid"`
	Comments    string `xml:"comments"`
	Link        string `xml:"link"`
	Category    string `xml:"category"`
	PubDate     string `xml:"pubDate"`
	Attr        []struct {
		Name  string `xml:"name,attr"`
		Value string `xml:"value,attr"`
	} `xml:"attr"`
}

func New(name string, url string, apikey string) TorznabIndexer {
	return TorznabIndexer{Name: name, Url: url, ApiKey: apikey}
}

func (torznabIndexer TorznabIndexer) GetTorrentForEpisode(show string, season int, episode int) ([]indexer.Torrent, error) {
	baseURL := torznabIndexer.Url

	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	httpClient := &http.Client{Transport: tr}

	urlObject, _ := url.ParseRequestURI(baseURL)

	var request *http.Request

	params := url.Values{}
	params.Add("apikey", torznabIndexer.ApiKey)
	params.Add("t", "tvsearch")
	params.Add("q", show)
	params.Add("season", strconv.Itoa(season))
	params.Add("episode", strconv.Itoa(episode))
	urlObject.RawQuery = params.Encode()

	request, err := http.NewRequest("GET", urlObject.String(), nil)
	if err != nil {
		return []indexer.Torrent{}, err
	}

	response, err := httpClient.Do(request)
	if err != nil {
		return []indexer.Torrent{}, err
	}

	body, readError := ioutil.ReadAll(response.Body)
	if readError != nil {
		return []indexer.Torrent{}, err
	}

	var searchResults TorrentSearchResults
	parseErr := xml.Unmarshal(body, &searchResults)
	if parseErr != nil {
		log.Debug(parseErr)
		return []indexer.Torrent{}, parseErr
	}

	// Construct Attributes map
	var results []indexer.Torrent
	for _, torrent := range searchResults.Torrents {
		resultTorrent := &indexer.Torrent{
			Title:       torrent.Title,
			Description: torrent.Description,
			Link:        torrent.Link,
		}

		resultTorrent.Attributes = make(map[string]string, len(torrent.Attr))
		for _, attr := range torrent.Attr {
			resultTorrent.Attributes[attr.Name] = attr.Value
		}
		results = append(results, *resultTorrent)
	}

	return results, nil
}

func (torznabIndexer TorznabIndexer) GetName() string {
	return torznabIndexer.Name
}
<commit_msg>Empty request results are now handled in torznab<commit_after>package torznab

import (
	"crypto/tls"
	"encoding/xml"
    "errors"
	"flemzerd/indexers"
	log "flemzerd/logging"
	//"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
)

type TorznabIndexer struct {
	Name   string
	Url    string
	ApiKey string
}

type TorrentSearchResults struct {
	Torrents []TorznabTorrent `xml:"channel>item"`
}

type TorznabTorrent struct {
	Title       string `xml:"title"`
	Description string `xml:"description"`
	Guid        string `xml:"guid"`
	Comments    string `xml:"comments"`
	Link        string `xml:"link"`
	Category    string `xml:"category"`
	PubDate     string `xml:"pubDate"`
	Attr        []struct {
		Name  string `xml:"name,attr"`
		Value string `xml:"value,attr"`
	} `xml:"attr"`
}

func New(name string, url string, apikey string) TorznabIndexer {
	return TorznabIndexer{Name: name, Url: url, ApiKey: apikey}
}

func (torznabIndexer TorznabIndexer) GetTorrentForEpisode(show string, season int, episode int) ([]indexer.Torrent, error) {
	baseURL := torznabIndexer.Url

	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	httpClient := &http.Client{Transport: tr}

	urlObject, _ := url.ParseRequestURI(baseURL)

	var request *http.Request

	params := url.Values{}
	params.Add("apikey", torznabIndexer.ApiKey)
	params.Add("t", "tvsearch")
	params.Add("q", show)
	params.Add("season", strconv.Itoa(season))
	params.Add("episode", strconv.Itoa(episode))
	urlObject.RawQuery = params.Encode()

	request, err := http.NewRequest("GET", urlObject.String(), nil)
	if err != nil {
		return []indexer.Torrent{}, err
	}

	response, err := httpClient.Do(request)
	if err != nil {
		return []indexer.Torrent{}, err
	}

	body, readError := ioutil.ReadAll(response.Body)
	if readError != nil {
		return []indexer.Torrent{}, err
	}

    if len(body) == 0 {
         return []indexer.Torrent{}, errors.New("Empty result")
    }

	var searchResults TorrentSearchResults
	parseErr := xml.Unmarshal(body, &searchResults)
	if parseErr != nil {
        log.Debug("ParseError: ", parseErr)
		return []indexer.Torrent{}, parseErr
	}

	// Construct Attributes map
	var results []indexer.Torrent
	for _, torrent := range searchResults.Torrents {
		resultTorrent := &indexer.Torrent{
			Title:       torrent.Title,
			Description: torrent.Description,
			Link:        torrent.Link,
		}

		resultTorrent.Attributes = make(map[string]string, len(torrent.Attr))
		for _, attr := range torrent.Attr {
			resultTorrent.Attributes[attr.Name] = attr.Value
		}
		results = append(results, *resultTorrent)
	}

	return results, nil
}

func (torznabIndexer TorznabIndexer) GetName() string {
	return torznabIndexer.Name
}
<|endoftext|>