<commit_before>package manta

import (
	"testing"

	"github.com/dotabuff/manta/dota"
	"github.com/golang/protobuf/proto"
	"github.com/stretchr/testify/assert"
)

const (
	HANDLE_NONE = uint32(2097151)
)

func TestReadProperties(t *testing.T) {
	assert := assert.New(t)

	scenarios := []struct {
		tableName   string                 // the name of the table, must have a sendtable fixture.
		run         bool                   // whether or not we run the test.
		debug       bool                   // whether or not we print debugging output.
		expectCount int                    // how many result entries we expect.
		expectKeys  map[string]interface{} // a map of entries we expect to be present.
	}{

		/*
			class id=571 name=CDOTATeam
			table id=586 name=CDOTATeam version=0
			 -> prop 0: type:uint8(727) name:m_iTeamNum(784) sendNode: (root)(734)
			 -> prop 1: type:CUtlVector< CHandle< CBasePlayer > >(1592) name:m_aPlayers(1593) sendNode: (root)(734)
			 -> prop 2: type:int32(706) name:m_iScore(1594) sendNode: (root)(734)
			 -> prop 3: type:int32(706) name:m_iRoundsWon(1595) sendNode: (root)(734)
			 -> prop 4: type:char[129](1596) name:m_szTeamname(1597) sendNode: (root)(734)
			 -> prop 5: type:int32(706) name:m_iHeroKills(1598) sendNode: (root)(734)
			 -> prop 6: type:int32(706) name:m_iTowerKills(1286) sendNode: (root)(734)
			 -> prop 7: type:int32(706) name:m_iBarracksKills(1599) sendNode: (root)(734)
			 -> prop 8: type:uint32(748) name:m_unTournamentTeamID(1600) sendNode: (root)(734)
			 -> prop 9: type:uint64(715) name:m_ulTeamLogo(1601) sendNode: (root)(734)
			 -> prop 10: type:uint64(715) name:m_ulTeamBaseLogo(1602) sendNode: (root)(734)
			 -> prop 11: type:uint64(715) name:m_ulTeamBannerLogo(1603) sendNode: (root)(734)
			 -> prop 12: type:bool(723) name:m_bTeamComplete(1604) sendNode: (root)(734)
			 -> prop 13: type:Color(774) name:m_CustomHealthbarColor(1605) sendNode: (root)(734)
			 -> prop 14: type:char[33](1606) name:m_szTag(1607) sendNode: (root)(734)
		*/
		{
			tableName:   "CDOTATeam",
			run:         true,
			debug:       false,
			expectCount: 15,
			expectKeys: map[string]interface{}{
				"m_iTeamNum":             uint64(0),
				"m_aPlayers":             uint32(0), // Most certainly wrong
				"m_iScore":               int32(0),
				"m_iRoundsWon":           int32(0),
				"m_szTeamname":           "Unassigned",
				"m_iHeroKills":           int32(0),
				"m_iTowerKills":          int32(0),
				"m_iBarracksKills":       int32(0),
				"m_unTournamentTeamID":   uint64(0),
				"m_ulTeamLogo":           uint64(0),
				"m_ulTeamBaseLogo":       uint64(0),
				"m_ulTeamBannerLogo":     uint64(0),
				"m_bTeamComplete":        false,
				"m_CustomHealthbarColor": int32(0),
				"m_szTag":                "",
			},
		},

		/*
			class id=559 name=CDOTAFogOfWarWasVisible
			table id=568 name=CDOTAFogOfWarWasVisible version=0
			 -> prop 0: type:uint64[1024](1355) name:m_bWasVisible(1356) sendNode: (root)(734)
		*/
		{
			tableName:   "CDOTAFogOfWarWasVisible",
			run:         true,
			debug:       false,
			expectCount: 1024,
			expectKeys: map[string]interface{}{
				"m_bWasVisible.0000": uint64(0),
				"m_bWasVisible.0001": uint64(0),
				"m_bWasVisible.0511": uint64(0),
				"m_bWasVisible.1023": uint64(0),
				"m_bWasVisible.1024": nil,
			},
		},

		/*
			class id=643 name=CRagdollManager
			table id=661 name=CRagdollManager version=0
			 -> prop 0: type:int8(718) name:m_iCurrentMaxRagdollCount(1982) sendNode: (root)(734)
		*/
		{
			tableName:   "CRagdollManager",
			run:         true,
			debug:       false,
			expectCount: 1,
			expectKeys: map[string]interface{}{
				"m_iCurrentMaxRagdollCount": int32(-1),
			},
		},

		/*
			class id=342 name=CDOTA_DataDire
			table id=351 name=CDOTA_DataDire version=0
			 -> prop 0: type:int32[10](1164) name:m_iReliableGold(1165) sendNode: (root)(734)
			 -> prop 1: type:int32[10](1164) name:m_iUnreliableGold(1166) sendNode: (root)(734)
			 -> prop 2: type:uint8(727) name:m_iTeamNum(784) sendNode: (root)(734)
			 -> prop 3: type:int32[30](1167) name:m_iStartingPositions(1168) sendNode: (root)(734)
			 -> prop 4: type:uint64[256](1169) name:m_bWorldTreeState(1170) sendNode: (root)(734)
		*/
		{
			tableName:   "CDOTA_DataDire",
			run:         true,
			debug:       false,
			expectCount: 428,
			expectKeys: map[string]interface{}{
				"m_vecDataTeam.0000.m_iUnreliableGold": int32(625),
				"m_iTeamNum":                           uint64(3),
				"m_bWorldTreeState.0000":               uint64(18446744073709551615),
				"m_bWorldTreeState.0110":               uint64(18446744073709551615),
				"m_bWorldTreeState.0128":               uint64(0),
				"m_bWorldTreeState.0129":               uint64(0),
				"m_bWorldTreeState.0255":               uint64(0),
			},
		},

		{
			tableName:   "CDOTA_DataRadiant",
			run:         true,
			debug:       false,
			expectCount: 428,
			expectKeys: map[string]interface{}{
				"m_vecDataTeam.0000.m_iUnreliableGold": int32(625),
				"m_iTeamNum":                           uint64(2),
				"m_bWorldTreeState.0000":               uint64(18446744073709551615),
				"m_bWorldTreeState.0110":               uint64(18446744073709551615),
				"m_bWorldTreeState.0128":               uint64(0),
				"m_bWorldTreeState.0129":               uint64(0),
				"m_bWorldTreeState.0255":               uint64(0),
			},
		},

		{
			tableName:   "CDOTA_DataSpectator",
			run:         true,
			debug:       false,
			expectCount: 284,
			expectKeys: map[string]interface{}{
				"m_hPrimaryRune":   HANDLE_NONE,
				"m_hSecondaryRune": HANDLE_NONE,
				"m_iNetWorth.0000": int32(625),
				"m_iNetWorth.0009": int32(625),
			},
		},

		{
			tableName:   "CDOTA_DataCustomTeam",
			run:         true,
			debug:       false,
			expectCount: 258,
			expectKeys: map[string]interface{}{
				"m_iTeamNum":             uint64(6),
				"m_bWorldTreeState.0000": uint64(18446744073709551615),
				"m_bWorldTreeState.0110": uint64(18446744073709551615),
				"m_bWorldTreeState.0128": uint64(0),
				"m_bWorldTreeState.0129": uint64(0),
				"m_bWorldTreeState.0255": uint64(0),
			},
		},

		{
			tableName:   "CFogController",
			run:         true,
			debug:       false,
			expectCount: 21,
			expectKeys: map[string]interface{}{
				"colorPrimary":         uint64(4294963108),
				"colorSecondary":       uint64(4294963108),
				"colorPrimaryLerpTo":   uint64(4294963108),
				"colorSecondaryLerpTo": uint64(4294963108),
				"enable":               true,
				"blend":                false,
				"m_bNoReflectionFog":   true,
			},
		},

		{
			tableName:   "CDOTASpectatorGraphManagerProxy",
			run:         true,
			debug:       false,
			expectCount: 412,
			expectKeys: map[string]interface{}{
				"CDOTASpectatorGraphManager.m_rgPlayerGraphData.0000": uint32(2097151),
				"CDOTASpectatorGraphManager.m_rgDireNetWorth.0063":    int32(0),
				"CDOTASpectatorGraphManager.m_nGoldGraphVersion":      int32(3),
			},
		},

		{
			tableName:   "CSpeechBubbleManager",
			run:         true,
			debug:       false,
			expectCount: 25, // WRONG, should carry the array index instead of overwriting. Needs some rework
			expectKeys: map[string]interface{}{
				"m_SpeechBubbles.0000.m_hNPC":       HANDLE_NONE,
				"m_SpeechBubbles.0000.m_flDuration": float32(0),
				"m_SpeechBubbles.0000.m_unOffsetX":  uint32(0),
				"m_SpeechBubbles.0000.m_unCount":    uint32(0),
			},
		},

		{
			tableName:   "CBaseEntity",
			run:         true,
			debug:       false,
			expectCount: 35,
			expectKeys: map[string]interface{}{
				"m_pEntity": true,
			},
		},

		{
			tableName:   "CBaseAnimating",
			run:         true,
			debug:       false,
			expectCount: 111,
			expectKeys: map[string]interface{}{
				"CRenderComponent":                      1,
				"CPhysicsComponent":                     1,
				"m_hEffectEntity":                       HANDLE_NONE,
				"CBodyComponentBaseAnimating.m_hParent": HANDLE_NONE,
				"m_hOwnerEntity":                        HANDLE_NONE,
			},
		},

		{
			tableName:   "CWorld",
			run:         true,
			debug:       false,
			expectCount: 139,
			expectKeys: map[string]interface{}{
				"m_iszOptimizedHeightFieldName": string(""),
				"m_hEffectEntity":               HANDLE_NONE,
				"m_pEntity":                     true,
			},
		},

		{
			tableName:   "CIngameEvent_TI5",
			run:         true,
			debug:       false,
			expectCount: 348,
			expectKeys: map[string]interface{}{
				"m_CompendiumChallengeInfo.0023.nType": 0,
			},
		},

		{
			tableName:   "CDOTAPlayer",
			run:         true,
			debug:       false,
			expectCount: 136,
			expectKeys: map[string]interface{}{
				"m_hViewEntity": HANDLE_NONE,
			},
		},
	}

	// Load our send tables
	m := &dota.CDemoSendTables{}
	if err := proto.Unmarshal(_read_fixture("send_tables/1731962898.pbmsg"), m); err != nil {
		panic(err)
	}

	// Retrieve the flattened field serializer
	fs := ParseSendTables(m, GetDefaultPropertySerializerTable())

	// Iterate through scenarios
	for _, s := range scenarios {
		// Load up a fixture
		buf := _read_fixture(_sprintf("instancebaseline/1731962898_%s.rawbuf", s.tableName))

		serializer := fs.Serializers[s.tableName][0]
		assert.NotNil(serializer)

		// Optionally skip
		if !s.run {
			continue
		}

		// Optionally disable debugging
		debugMode = s.debug

		// Read properties
		r := NewReader(buf)
		props := ReadProperties(r, serializer)
		assert.Equal(s.expectCount, len(props))

		for k, v := range s.expectKeys {
			assert.EqualValues(v, props[k])
		}

		// There shouldn't be more than 8 bits left in the buffer
		_debugf("Remaining bits %v", r.remBits())
		assert.True(r.remBits() < 8)
	}
}
<commit_msg>Handle is read as a varint<commit_after>package manta

import (
	"testing"

	"github.com/dotabuff/manta/dota"
	"github.com/golang/protobuf/proto"
	"github.com/stretchr/testify/assert"
)

const (
	HANDLE_NONE = uint32(16777215)
)

func TestReadProperties(t *testing.T) {
	assert := assert.New(t)

	scenarios := []struct {
		tableName   string                 // the name of the table, must have a sendtable fixture.
		run         bool                   // whether or not we run the test.
		debug       bool                   // whether or not we print debugging output.
		expectCount int                    // how many result entries we expect.
		expectKeys  map[string]interface{} // a map of entries we expect to be present.
	}{

		/*
			class id=571 name=CDOTATeam
			table id=586 name=CDOTATeam version=0
			 -> prop 0: type:uint8(727) name:m_iTeamNum(784) sendNode: (root)(734)
			 -> prop 1: type:CUtlVector< CHandle< CBasePlayer > >(1592) name:m_aPlayers(1593) sendNode: (root)(734)
			 -> prop 2: type:int32(706) name:m_iScore(1594) sendNode: (root)(734)
			 -> prop 3: type:int32(706) name:m_iRoundsWon(1595) sendNode: (root)(734)
			 -> prop 4: type:char[129](1596) name:m_szTeamname(1597) sendNode: (root)(734)
			 -> prop 5: type:int32(706) name:m_iHeroKills(1598) sendNode: (root)(734)
			 -> prop 6: type:int32(706) name:m_iTowerKills(1286) sendNode: (root)(734)
			 -> prop 7: type:int32(706) name:m_iBarracksKills(1599) sendNode: (root)(734)
			 -> prop 8: type:uint32(748) name:m_unTournamentTeamID(1600) sendNode: (root)(734)
			 -> prop 9: type:uint64(715) name:m_ulTeamLogo(1601) sendNode: (root)(734)
			 -> prop 10: type:uint64(715) name:m_ulTeamBaseLogo(1602) sendNode: (root)(734)
			 -> prop 11: type:uint64(715) name:m_ulTeamBannerLogo(1603) sendNode: (root)(734)
			 -> prop 12: type:bool(723) name:m_bTeamComplete(1604) sendNode: (root)(734)
			 -> prop 13: type:Color(774) name:m_CustomHealthbarColor(1605) sendNode: (root)(734)
			 -> prop 14: type:char[33](1606) name:m_szTag(1607) sendNode: (root)(734)
		*/
		{
			tableName:   "CDOTATeam",
			run:         true,
			debug:       false,
			expectCount: 15,
			expectKeys: map[string]interface{}{
				"m_iTeamNum":             uint64(0),
				"m_aPlayers":             uint32(0), // Most certainly wrong
				"m_iScore":               int32(0),
				"m_iRoundsWon":           int32(0),
				"m_szTeamname":           "Unassigned",
				"m_iHeroKills":           int32(0),
				"m_iTowerKills":          int32(0),
				"m_iBarracksKills":       int32(0),
				"m_unTournamentTeamID":   uint64(0),
				"m_ulTeamLogo":           uint64(0),
				"m_ulTeamBaseLogo":       uint64(0),
				"m_ulTeamBannerLogo":     uint64(0),
				"m_bTeamComplete":        false,
				"m_CustomHealthbarColor": int32(0),
				"m_szTag":                "",
			},
		},

		/*
			class id=559 name=CDOTAFogOfWarWasVisible
			table id=568 name=CDOTAFogOfWarWasVisible version=0
			 -> prop 0: type:uint64[1024](1355) name:m_bWasVisible(1356) sendNode: (root)(734)
		*/
		{
			tableName:   "CDOTAFogOfWarWasVisible",
			run:         true,
			debug:       false,
			expectCount: 1024,
			expectKeys: map[string]interface{}{
				"m_bWasVisible.0000": uint64(0),
				"m_bWasVisible.0001": uint64(0),
				"m_bWasVisible.0511": uint64(0),
				"m_bWasVisible.1023": uint64(0),
				"m_bWasVisible.1024": nil,
			},
		},

		/*
			class id=643 name=CRagdollManager
			table id=661 name=CRagdollManager version=0
			 -> prop 0: type:int8(718) name:m_iCurrentMaxRagdollCount(1982) sendNode: (root)(734)
		*/
		{
			tableName:   "CRagdollManager",
			run:         true,
			debug:       false,
			expectCount: 1,
			expectKeys: map[string]interface{}{
				"m_iCurrentMaxRagdollCount": int32(-1),
			},
		},

		/*
			class id=342 name=CDOTA_DataDire
			table id=351 name=CDOTA_DataDire version=0
			 -> prop 0: type:int32[10](1164) name:m_iReliableGold(1165) sendNode: (root)(734)
			 -> prop 1: type:int32[10](1164) name:m_iUnreliableGold(1166) sendNode: (root)(734)
			 -> prop 2: type:uint8(727) name:m_iTeamNum(784) sendNode: (root)(734)
			 -> prop 3: type:int32[30](1167) name:m_iStartingPositions(1168) sendNode: (root)(734)
			 -> prop 4: type:uint64[256](1169) name:m_bWorldTreeState(1170) sendNode: (root)(734)
		*/
		{
			tableName:   "CDOTA_DataDire",
			run:         true,
			debug:       false,
			expectCount: 428,
			expectKeys: map[string]interface{}{
				"m_vecDataTeam.0000.m_iUnreliableGold": int32(625),
				"m_iTeamNum":                           uint64(3),
				"m_bWorldTreeState.0000":               uint64(18446744073709551615),
				"m_bWorldTreeState.0110":               uint64(18446744073709551615),
				"m_bWorldTreeState.0128":               uint64(0),
				"m_bWorldTreeState.0129":               uint64(0),
				"m_bWorldTreeState.0255":               uint64(0),
			},
		},

		{
			tableName:   "CDOTA_DataRadiant",
			run:         true,
			debug:       false,
			expectCount: 428,
			expectKeys: map[string]interface{}{
				"m_vecDataTeam.0000.m_iUnreliableGold": int32(625),
				"m_iTeamNum":                           uint64(2),
				"m_bWorldTreeState.0000":               uint64(18446744073709551615),
				"m_bWorldTreeState.0110":               uint64(18446744073709551615),
				"m_bWorldTreeState.0128":               uint64(0),
				"m_bWorldTreeState.0129":               uint64(0),
				"m_bWorldTreeState.0255":               uint64(0),
			},
		},

		{
			tableName:   "CDOTA_DataSpectator",
			run:         true,
			debug:       false,
			expectCount: 284,
			expectKeys: map[string]interface{}{
				"m_hPrimaryRune":   HANDLE_NONE,
				"m_hSecondaryRune": HANDLE_NONE,
				"m_iNetWorth.0000": int32(625),
				"m_iNetWorth.0009": int32(625),
			},
		},

		{
			tableName:   "CDOTA_DataCustomTeam",
			run:         true,
			debug:       false,
			expectCount: 258,
			expectKeys: map[string]interface{}{
				"m_iTeamNum":             uint64(6),
				"m_bWorldTreeState.0000": uint64(18446744073709551615),
				"m_bWorldTreeState.0110": uint64(18446744073709551615),
				"m_bWorldTreeState.0128": uint64(0),
				"m_bWorldTreeState.0129": uint64(0),
				"m_bWorldTreeState.0255": uint64(0),
			},
		},

		{
			tableName:   "CFogController",
			run:         true,
			debug:       false,
			expectCount: 21,
			expectKeys: map[string]interface{}{
				"colorPrimary":         uint64(4294963108),
				"colorSecondary":       uint64(4294963108),
				"colorPrimaryLerpTo":   uint64(4294963108),
				"colorSecondaryLerpTo": uint64(4294963108),
				"enable":               true,
				"blend":                false,
				"m_bNoReflectionFog":   true,
			},
		},

		{
			tableName:   "CDOTASpectatorGraphManagerProxy",
			run:         true,
			debug:       false,
			expectCount: 412,
			expectKeys: map[string]interface{}{
				"CDOTASpectatorGraphManager.m_rgPlayerGraphData.0000": HANDLE_NONE,
				"CDOTASpectatorGraphManager.m_rgDireNetWorth.0063":    int32(0),
				"CDOTASpectatorGraphManager.m_nGoldGraphVersion":      int32(3),
			},
		},

		{
			tableName:   "CSpeechBubbleManager",
			run:         true,
			debug:       false,
			expectCount: 25, // WRONG, should carry the array index instead of overwriting. Needs some rework
			expectKeys: map[string]interface{}{
				"m_SpeechBubbles.0000.m_hNPC":       HANDLE_NONE,
				"m_SpeechBubbles.0000.m_flDuration": float32(0),
				"m_SpeechBubbles.0000.m_unOffsetX":  uint32(0),
				"m_SpeechBubbles.0000.m_unCount":    uint32(0),
			},
		},

		{
			tableName:   "CBaseEntity",
			run:         true,
			debug:       false,
			expectCount: 35,
			expectKeys: map[string]interface{}{
				"m_pEntity": true,
			},
		},

		{
			tableName:   "CBaseAnimating",
			run:         true,
			debug:       false,
			expectCount: 111,
			expectKeys: map[string]interface{}{
				"CRenderComponent":                      1,
				"CPhysicsComponent":                     1,
				"m_hEffectEntity":                       HANDLE_NONE,
				"CBodyComponentBaseAnimating.m_hParent": HANDLE_NONE,
				"m_hOwnerEntity":                        HANDLE_NONE,
			},
		},

		{
			tableName:   "CWorld",
			run:         true,
			debug:       false,
			expectCount: 139,
			expectKeys: map[string]interface{}{
				"m_iszOptimizedHeightFieldName": string(""),
				"m_hEffectEntity":               HANDLE_NONE,
				"m_pEntity":                     true,
			},
		},

		{
			tableName:   "CIngameEvent_TI5",
			run:         true,
			debug:       false,
			expectCount: 348,
			expectKeys: map[string]interface{}{
				"m_CompendiumChallengeInfo.0023.nType": 0,
			},
		},

		{
			tableName:   "CDOTAPlayer",
			run:         true,
			debug:       false,
			expectCount: 136,
			expectKeys: map[string]interface{}{
				"m_hViewEntity": HANDLE_NONE,
			},
		},
	}

	// Load our send tables
	m := &dota.CDemoSendTables{}
	if err := proto.Unmarshal(_read_fixture("send_tables/1731962898.pbmsg"), m); err != nil {
		panic(err)
	}

	// Retrieve the flattened field serializer
	fs := ParseSendTables(m, GetDefaultPropertySerializerTable())

	// Iterate through scenarios
	for _, s := range scenarios {
		// Load up a fixture
		buf := _read_fixture(_sprintf("instancebaseline/1731962898_%s.rawbuf", s.tableName))

		serializer := fs.Serializers[s.tableName][0]
		assert.NotNil(serializer)

		// Optionally skip
		if !s.run {
			continue
		}

		// Optionally disable debugging
		debugMode = s.debug
		if debugMode {
			debugLevel = 10
		} else {
			debugLevel = 0
		}

		// Read properties
		r := NewReader(buf)
		props := ReadProperties(r, serializer)
		assert.Equal(s.expectCount, len(props))

		for k, v := range s.expectKeys {
			assert.EqualValues(v, props[k])
		}

		// There shouldn't be more than 8 bits left in the buffer
		_debugf("Remaining bits %v", r.remBits())
		assert.True(r.remBits() < 8)
	}
}
<|endoftext|>


<commit_before>package server

import (
	"flag"
	"fmt"
	"github.com/gorilla/mux"
	"image"
	"io"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"log"
	"net/http"
	"time"
)

const JpegMaximumQuality = 100
const ImageCacheDuration = 315360000

var Connection *mgo.Session
var Configuration *Config

// VarsHandler is a simple wrapper so the request params can be injected into the main handler
type VarsHandler func(http.ResponseWriter, *http.Request, *ServerConfiguration)

// ServeHTTP wraps the imageHandler function and validates request parameters
// in order to create a ServerConfiguration object
func (h VarsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)

	requestConfig, validateError := CreateConfigurationFromVars(r, vars)

	if validateError != nil {
		log.Printf("%d invalid request parameters given.\n", http.StatusNotFound)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	h(w, r, requestConfig)
}

// storeImage saves the image
func storeImage(targetImage *mgo.GridFile, imageData image.Image, imageFormat string, originalImage *mgo.GridFile, entry *Entry) error {
	width := imageData.Bounds().Dx()
	height := imageData.Bounds().Dy()
	originalRef := mgo.DBRef{"fs.files", originalImage.Id(), ""}

	metadata := bson.M{
		"width":            width,
		"height":           height,
		"original":         originalRef,
		"originalFilename": originalImage.Name(),
		"resizeType":       entry.Type,
		"size":             fmt.Sprintf("%dx%d", width, height)}

	targetImage.SetContentType(fmt.Sprintf("image/%s", imageFormat))
	targetImage.SetMeta(metadata)

	targetImage.Close()

	return nil
}

// isModified returns true if the file must be delivered, false otherwise.
func isModified(file *mgo.GridFile, header *http.Header) bool {
	md5 := file.MD5()
	modifiedHeader := header.Get("If-Modified-Since")
	modifiedTime := time.Now()

	if modifiedHeader != "" {
		modifiedTime, _ = time.Parse(time.RFC1123, modifiedHeader)
	}

	// normalize upload date to use the same format as the browser
	uploadDate, _ := time.Parse(time.RFC1123, file.UploadDate().Format(time.RFC1123))

	if header.Get("Cache-Control") == "no-cache" {
		log.Printf("Is modified, because caching not enabled.")
		return true
	}

	if uploadDate.After(modifiedTime) {
		log.Printf("Is modified, because upload date after modified date.\n")
		return true
	}

	if md5 != header.Get("If-None-Match") {
		log.Printf("Is modified, because md5 mismatch. %s != %s", md5, header.Get("If-None-Match"))
		return true
	}

	return false
}

// setCacheHeaders sets the cache headers into the http.ResponseWriter
func setCacheHeaders(file *mgo.GridFile, w http.ResponseWriter) {
	w.Header().Set("Etag", file.MD5())
	w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d", ImageCacheDuration))
	d, _ := time.ParseDuration(fmt.Sprintf("%ds", ImageCacheDuration))

	expires := file.UploadDate().Add(d)

	w.Header().Set("Last-Modified", file.UploadDate().Format(time.RFC1123))
	w.Header().Set("Expires", expires.Format(time.RFC1123))
	w.Header().Set("Date", file.UploadDate().Format(time.RFC1123))
}

// imageHandler the main handler
func imageHandler(w http.ResponseWriter, r *http.Request, requestConfig *ServerConfiguration) {
	log.Printf("Request on %s", r.URL)

	gridfs := Connection.DB(requestConfig.Database).GridFS("fs")

	resizeEntry, _ := Configuration.GetEntryByName(requestConfig.FormatName)
	foundImage, _ := FindImageByParentFilename(requestConfig.Filename, resizeEntry, gridfs)

	// case that we do not want resizing and did not find any image
	if foundImage == nil && resizeEntry == nil {
		log.Printf("%d invalid request parameters given.\n", http.StatusNotFound)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// we found a image but did not want resizing
	if foundImage != nil {
		if !isModified(foundImage, &r.Header) {
			w.WriteHeader(http.StatusNotModified)
			log.Printf("%d Returning cached image.\n", http.StatusNotModified)
			return
		}

		setCacheHeaders(foundImage, w)

		io.Copy(w, foundImage)
		foundImage.Close()
		log.Printf("%d Image found, no resizing.\n", http.StatusOK)
		return
	}

	if foundImage == nil && resizeEntry != nil {
		// generate new image
		foundImage, _ = FindImageByParentFilename(requestConfig.Filename, nil, gridfs)

		if foundImage == nil {
			log.Printf("%d Could not find original image.\n", http.StatusNotFound)
			w.WriteHeader(http.StatusNotFound)
			return
		}

		resizedImage, imageFormat, imageErr := ResizeImageFromGridfs(foundImage, resizeEntry)

		// in this case, resizing for this image does not work, therefore, we at least return the original image
		if imageErr != nil {

			// this might be a problem at the moment, go does not support interlaced pngs
			// http://code.google.com/p/go/issues/detail?id=6293
			// at the moment, we return a not found...
			log.Printf("%d image could not be decoded.\n", http.StatusNotFound)
			w.WriteHeader(http.StatusNotFound)
			return
		}

		// return the image to the client if all cache headers could be set
		targetfile, _ := gridfs.Create(GetRandomFilename(imageFormat))
		encodeErr := EncodeImage(targetfile, *resizedImage, imageFormat)

		if encodeErr != nil {
			log.Fatalf(imageErr.Error())
			w.WriteHeader(http.StatusBadRequest)
			log.Printf("%d image could not be encoded.\n", http.StatusBadRequest)
			return
		}

		storeErr := storeImage(targetfile, *resizedImage, imageFormat, foundImage, resizeEntry)

		if storeErr != nil {
			log.Fatalf(imageErr.Error())
			w.WriteHeader(http.StatusBadRequest)
			log.Printf("%d image could not be saved.\n", http.StatusBadRequest)
			return
		}

		fp, readErr := gridfs.Open(targetfile.Name())

		if fp != nil {
			if !isModified(fp, &r.Header) {
				w.WriteHeader(http.StatusNotModified)
				log.Printf("%d Returning cached image.\n", http.StatusNotModified)
				return
			}

			setCacheHeaders(fp, w)

			io.Copy(w, fp)
			fp.Close()
			log.Printf("%d image succesfully resized and returned.\n", http.StatusOK)
		} else {
			log.Printf(readErr.Error())
			w.WriteHeader(http.StatusBadRequest)
			return
		}
	}
}

//Deliver is the startup method that parses configuration files and opens the mongo connection
func Deliver() int {
	configurationFilepath := flag.String("config", "configuration.json", "path to the configuration file")
	serverPort := flag.Int("port", 8000, "the server port where we will serve images")
	host := flag.String("host", "localhost:27017", "the database host with an optional port, localhost would suffice")

	flag.Parse()

	var err error

	Configuration, err = CreateConfigFromFile(*configurationFilepath)

	if err != nil {
		fmt.Printf("Error %s\n", err)
		return -2
	}

	fmt.Printf("Server started. Listening on %d database host is %s\n", *serverPort, *host)

	// in order to simple configure the image server in the proxy configuration of nginx
	// we will be getting every database variable from the request
	serverRoute := "/{database}/{filename}"

	Connection, err = mgo.Dial(*host)

	if err != nil {
		log.Fatal("Cannot connect to database")
		return -1
	}

	Connection.SetMode(mgo.Eventual, true)
	Connection.SetSyncTimeout(0)

	r := mux.NewRouter()
	r.HandleFunc("/", welcomeHandler)
	r.Handle(serverRoute, VarsHandler(imageHandler))

	http.Handle("/", r)

	err = http.ListenAndServe(fmt.Sprintf(":%d", *serverPort), nil)

	if err != nil {
		log.Fatal(err)
		return -1
	}

	return 0
}

// just a static welcome handler
func welcomeHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "")
	fmt.Fprintf(w, "
Image Server.
")
	fmt.Fprintf(w, "")
}
<commit_msg>Renames one method<commit_after>package server

import (
	"flag"
	"fmt"
	"github.com/gorilla/mux"
	"image"
	"io"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"log"
	"net/http"
	"time"
)

const JpegMaximumQuality = 100
const ImageCacheDuration = 315360000

var Connection *mgo.Session
var Configuration *Config

// VarsHandler is a simple wrapper so the request params can be injected into the main handler
type VarsHandler func(http.ResponseWriter, *http.Request, *ServerConfiguration)

// ServeHTTP wraps the imageHandler function and validates request parameters
// in order to create a ServerConfiguration object
func (h VarsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)

	requestConfig, validateError := CreateConfigurationFromVars(r, vars)

	if validateError != nil {
		log.Printf("%d invalid request parameters given.\n", http.StatusNotFound)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	h(w, r, requestConfig)
}

// addImageMetaData adds data to the image the image
func addImageMetaData(targetImage *mgo.GridFile, imageData image.Image, imageFormat string, originalImage *mgo.GridFile, entry *Entry) {
	width := imageData.Bounds().Dx()
	height := imageData.Bounds().Dy()
	originalRef := mgo.DBRef{"fs.files", originalImage.Id(), ""}

	metadata := bson.M{
		"width":            width,
		"height":           height,
		"original":         originalRef,
		"originalFilename": originalImage.Name(),
		"resizeType":       entry.Type,
		"size":             fmt.Sprintf("%dx%d", width, height)}

	targetImage.SetContentType(fmt.Sprintf("image/%s", imageFormat))
	targetImage.SetMeta(metadata)
}

// isModified returns true if the file must be delivered, false otherwise.
func isModified(file *mgo.GridFile, header *http.Header) bool {
	md5 := file.MD5()
	modifiedHeader := header.Get("If-Modified-Since")
	modifiedTime := time.Now()

	if modifiedHeader != "" {
		modifiedTime, _ = time.Parse(time.RFC1123, modifiedHeader)
	}

	// normalize upload date to use the same format as the browser
	uploadDate, _ := time.Parse(time.RFC1123, file.UploadDate().Format(time.RFC1123))

	if header.Get("Cache-Control") == "no-cache" {
		log.Printf("Is modified, because caching not enabled.")
		return true
	}

	if uploadDate.After(modifiedTime) {
		log.Printf("Is modified, because upload date after modified date.\n")
		return true
	}

	if md5 != header.Get("If-None-Match") {
		log.Printf("Is modified, because md5 mismatch. %s != %s", md5, header.Get("If-None-Match"))
		return true
	}

	return false
}

// setCacheHeaders sets the cache headers into the http.ResponseWriter
func setCacheHeaders(file *mgo.GridFile, w http.ResponseWriter) {
	w.Header().Set("Etag", file.MD5())
	w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d", ImageCacheDuration))
	d, _ := time.ParseDuration(fmt.Sprintf("%ds", ImageCacheDuration))

	expires := file.UploadDate().Add(d)

	w.Header().Set("Last-Modified", file.UploadDate().Format(time.RFC1123))
	w.Header().Set("Expires", expires.Format(time.RFC1123))
	w.Header().Set("Date", file.UploadDate().Format(time.RFC1123))
}

// imageHandler the main handler
func imageHandler(w http.ResponseWriter, r *http.Request, requestConfig *ServerConfiguration) {
	log.Printf("Request on %s", r.URL)

	gridfs := Connection.DB(requestConfig.Database).GridFS("fs")

	resizeEntry, _ := Configuration.GetEntryByName(requestConfig.FormatName)
	foundImage, _ := FindImageByParentFilename(requestConfig.Filename, resizeEntry, gridfs)

	// case that we do not want resizing and did not find any image
	if foundImage == nil && resizeEntry == nil {
		log.Printf("%d invalid request parameters given.\n", http.StatusNotFound)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// we found a image but did not want resizing
	if foundImage != nil {
		if !isModified(foundImage, &r.Header) {
			w.WriteHeader(http.StatusNotModified)
			log.Printf("%d Returning cached image.\n", http.StatusNotModified)
			return
		}

		setCacheHeaders(foundImage, w)

		io.Copy(w, foundImage)
		foundImage.Close()
		log.Printf("%d Image found, no resizing.\n", http.StatusOK)
		return
	}

	if foundImage == nil && resizeEntry != nil {
		// generate new image
		foundImage, _ = FindImageByParentFilename(requestConfig.Filename, nil, gridfs)

		if foundImage == nil {
			log.Printf("%d Could not find original image.\n", http.StatusNotFound)
			w.WriteHeader(http.StatusNotFound)
			return
		}

		resizedImage, imageFormat, imageErr := ResizeImageFromGridfs(foundImage, resizeEntry)

		// in this case, resizing for this image does not work, therefore, we at least return the original image
		if imageErr != nil {

			// this might be a problem at the moment, go does not support interlaced pngs
			// http://code.google.com/p/go/issues/detail?id=6293
			// at the moment, we return a not found...
			log.Printf("%d image could not be decoded.\n", http.StatusNotFound)
			w.WriteHeader(http.StatusNotFound)
			return
		}

		// return the image to the client if all cache headers could be set
		targetfile, _ := gridfs.Create(GetRandomFilename(imageFormat))
		encodeErr := EncodeImage(targetfile, *resizedImage, imageFormat)

		if encodeErr != nil {
			log.Fatalf(imageErr.Error())
			w.WriteHeader(http.StatusBadRequest)
			log.Printf("%d image could not be encoded.\n", http.StatusBadRequest)
			return
		}

		addImageMetaData(targetfile, *resizedImage, imageFormat, foundImage, resizeEntry)

		targetfile.Close()

		fp, readErr := gridfs.Open(targetfile.Name())

		if fp != nil {
			if !isModified(fp, &r.Header) {
				w.WriteHeader(http.StatusNotModified)
				log.Printf("%d Returning cached image.\n", http.StatusNotModified)
				return
			}

			setCacheHeaders(fp, w)

			io.Copy(w, fp)
			fp.Close()
			log.Printf("%d image succesfully resized and returned.\n", http.StatusOK)
		} else {
			log.Printf(readErr.Error())
			w.WriteHeader(http.StatusBadRequest)
			return
		}
	}
}

//Deliver is the startup method that parses configuration files and opens the mongo connection
func Deliver() int {
	configurationFilepath := flag.String("config", "configuration.json", "path to the configuration file")
	serverPort := flag.Int("port", 8000, "the server port where we will serve images")
	host := flag.String("host", "localhost:27017", "the database host with an optional port, localhost would suffice")

	flag.Parse()

	var err error

	Configuration, err = CreateConfigFromFile(*configurationFilepath)

	if err != nil {
		fmt.Printf("Error %s\n", err)
		return -2
	}

	fmt.Printf("Server started. Listening on %d database host is %s\n", *serverPort, *host)

	// in order to simple configure the image server in the proxy configuration of nginx
	// we will be getting every database variable from the request
	serverRoute := "/{database}/{filename}"

	Connection, err = mgo.Dial(*host)

	if err != nil {
		log.Fatal("Cannot connect to database")
		return -1
	}

	Connection.SetMode(mgo.Eventual, true)
	Connection.SetSyncTimeout(0)

	r := mux.NewRouter()
	r.HandleFunc("/", welcomeHandler)
	r.Handle(serverRoute, VarsHandler(imageHandler))

	http.Handle("/", r)

	err = http.ListenAndServe(fmt.Sprintf(":%d", *serverPort), nil)

	if err != nil {
		log.Fatal(err)
		return -1
	}

	return 0
}

// just a static welcome handler
func welcomeHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "")
	fmt.Fprintf(w, "
Image Server.
")
	fmt.Fprintf(w, "")
}
<|endoftext|>


<commit_before>package jsonstore

import (
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"testing"
)

type Human struct {
	Name   string
	Height float64
}

func testFile() *os.File {
	f, err := ioutil.TempFile(".", "jsonstore")
	if err != nil {
		panic(err)
	}
	return f
}

func TestOpen(t *testing.T) {
	f := testFile()
	defer os.Remove(f.Name())
	ioutil.WriteFile(f.Name(), []byte(`{"hello":"world"}`), 0644)
	ks, err := Open(f.Name())
	if err != nil {
		t.Error(err)
	}
	if len(ks.Data) != 1 {
		t.Errorf("expected %d got %d", 1, len(ks.Data))
	}
	if world, ok := ks.Data["hello"]; !ok || string(world) != `world` {
		t.Errorf("expected %s got %s", "world", world)
	}
}

func TestGeneral(t *testing.T) {
	f := testFile()
	defer os.Remove(f.Name())
	ks := new(JSONStore)
	err := ks.Set("hello", "world")
	if err != nil {
		t.Error(err)
	}
	if err = Save(ks, f.Name()); err != nil {
		t.Error(err)
	}

	ks2, _ := Open(f.Name())
	var a string
	var b string
	ks.Get("hello", &a)
	ks2.Get("hello", &b)
	if a != b {
		t.Errorf("expected '%s' got '%s'", a, b)
	}

	// Set a object, using a Gzipped JSON
	type Human struct {
		Name   string
		Height float64
	}
	ks.Set("human:1", Human{"Dante", 5.4})
	Save(ks, "test2.json.gz")
	ks2, _ = Open("test2.json.gz")
	var human Human
	ks2.Get("human:1", &human)
	if human.Height != 5.4 {
		t.Errorf("expected '%v', got '%v'", Human{"Dante", 5.4}, human)
	}
}

func TestRegex(t *testing.T) {
	f := testFile()
	defer os.Remove(f.Name())
	ks := new(JSONStore)
	ks.Set("hello:1", "world1")
	ks.Set("hello:2", "world2")
	ks.Set("hello:3", "world3")
	ks.Set("world:1", "hello1")
	if len(ks.GetAll(regexp.MustCompile(`hello`))) != len(ks.Keys())-1 {
		t.Errorf("Problem getting all")
	}
}

func BenchmarkOpenBig(b *testing.B) {
	f := testFile()
	defer os.Remove(f.Name())
	ks := new(JSONStore)
	for i := 1; i < 1000; i++ {
		ks.Set("hello:"+strconv.Itoa(i), "world"+strconv.Itoa(i))
	}
	Save(ks, f.Name())

	var err error
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		ks, err = Open(f.Name())
		if err != nil {
			panic(err)
		}
	}
	Save(ks, f.Name())
}

func BenchmarkOpenOldBig(b *testing.B) {
	f := testFile()
	defer os.Remove(f.Name())
	ks := new(JSONStore)
	for i := 1; i < 1000; i++ {
		ks.Set("hello:"+strconv.Itoa(i), "world"+strconv.Itoa(i))
	}
	Save(ks, f.Name())

	var err error
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		ks, err = OpenOld(f.Name())
		if err != nil {
			panic(err)
		}
	}
	Save(ks, f.Name())
}

func BenchmarkOpenSmall(b *testing.B) {
	f := testFile()
	defer os.Remove(f.Name())
	ks := new(JSONStore)
	for i := 1; i < 10; i++ {
		ks.Set("hello:"+strconv.Itoa(i), "world"+strconv.Itoa(i))
	}
	Save(ks, f.Name())

	var err error
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		ks, err = Open(f.Name())
		if err != nil {
			panic(err)
		}
	}
	Save(ks, f.Name())
}

func BenchmarkOpenOldSmall(b *testing.B) {
	f := testFile()
	defer os.Remove(f.Name())
	ks := new(JSONStore)
	for i := 1; i < 10; i++ {
		ks.Set("hello:"+strconv.Itoa(i), "world"+strconv.Itoa(i))
	}
	Save(ks, f.Name())

	var err error
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		ks, err = OpenOld(f.Name())
		if err != nil {
			panic(err)
		}
	}
	Save(ks, f.Name())
}

func BenchmarkOpen(b *testing.B) {
	f := testFile()
	defer os.Remove(f.Name())
	ks := new(JSONStore)
	for i := 1; i < 100; i++ {
		ks.Set("hello:"+strconv.Itoa(i), "world"+strconv.Itoa(i))
	}
	Save(ks, f.Name())

	var err error
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		ks, err = Open(f.Name())
		if err != nil {
			panic(err)
		}
	}
	Save(ks, f.Name())
}

func BenchmarkGet(b *testing.B) {
	ks := new(JSONStore)
	err := ks.Set("human:1", Human{"Dante", 5.4})
	if err != nil {
		panic(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		var human Human
		ks.Get("human:1", &human)
	}
}

func BenchmarkSet(b *testing.B) {
	ks := new(JSONStore)
	b.ResetTimer()
	// set a key to any object you want
	for i := 0; i < b.N; i++ {
		err := ks.Set("human:1", Human{"Dante", 5.4})
		if err != nil {
			panic(err)
		}
	}
}

func BenchmarkSave(b *testing.B) {
	ks := new(JSONStore)
	ks.Set("data", 1234)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		Save(ks, "benchmark.json.gz")
	}
}
<commit_msg>Added test for incorrectly written file<commit_after>package jsonstore

import (
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"testing"
)

type Human struct {
	Name   string
	Height float64
}

func testFile() *os.File {
	f, err := ioutil.TempFile(".", "jsonstore")
	if err != nil {
		panic(err)
	}
	return f
}

func TestOpen(t *testing.T) {
	f := testFile()
	defer os.Remove(f.Name())
	ioutil.WriteFile(f.Name(), []byte(`{"hello":"world"}`), 0644)
	ks, err := Open(f.Name())
	if err != nil {
		t.Error(err)
	}
	if len(ks.Data) != 1 {
		t.Errorf("expected %d got %d", 1, len(ks.Data))
	}
	if world, ok := ks.Data["hello"]; !ok || string(world) != `world` {
		t.Errorf("expected %s got %s", "world", world)
	}
}

func TestGeneral(t *testing.T) {
	f := testFile()
	defer os.Remove(f.Name())
	ks := new(JSONStore)
	err := ks.Set("hello", "world")
	if err != nil {
		t.Error(err)
	}
	if err = Save(ks, f.Name()); err != nil {
		t.Error(err)
	}

	ks2, _ := Open(f.Name())
	var a string
	var b string
	ks.Get("hello", &a)
	ks2.Get("hello", &b)
	if a != b {
		t.Errorf("expected '%s' got '%s'", a, b)
	}

	// Set a object, using a Gzipped JSON
	type Human struct {
		Name   string
		Height float64
	}
	ks.Set("human:1", Human{"Dante", 5.4})
	Save(ks, "test2.json.gz")
	ks2, err = Open("test2.json.gz")
	if err != nil {
		t.Errorf(err.Error())
		os.Exit(-1)
	}
	var human Human
	ks2.Get("human:1", &human)
	if human.Height != 5.4 {
		t.Errorf("expected '%v', got '%v'", Human{"Dante", 5.4}, human)
	}
}

func TestRegex(t *testing.T) {
	f := testFile()
	defer os.Remove(f.Name())
	ks := new(JSONStore)
	ks.Set("hello:1", "world1")
	ks.Set("hello:2", "world2")
	ks.Set("hello:3", "world3")
	ks.Set("world:1", "hello1")
	if len(ks.GetAll(regexp.MustCompile(`hello`))) != len(ks.Keys())-1 {
		t.Errorf("Problem getting all")
	}
}

func BenchmarkOpenBig(b *testing.B) {
	f := testFile()
	defer os.Remove(f.Name())
	ks := new(JSONStore)
	for i := 1; i < 1000; i++ {
		ks.Set("hello:"+strconv.Itoa(i), "world"+strconv.Itoa(i))
	}
	Save(ks, f.Name())

	var err error
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		ks, err = Open(f.Name())
		if err != nil {
			panic(err)
		}
	}
	Save(ks, f.Name())
}

func BenchmarkOpenOldBig(b *testing.B) {
	f := testFile()
	defer os.Remove(f.Name())
	ks := new(JSONStore)
	for i := 1; i < 1000; i++ {
		ks.Set("hello:"+strconv.Itoa(i), "world"+strconv.Itoa(i))
	}
	Save(ks, f.Name())

	var err error
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		ks, err = OpenOld(f.Name())
		if err != nil {
			panic(err)
		}
	}
	Save(ks, f.Name())
}

func BenchmarkOpenSmall(b *testing.B) {
	f := testFile()
	defer os.Remove(f.Name())
	ks := new(JSONStore)
	for i := 1; i < 10; i++ {
		ks.Set("hello:"+strconv.Itoa(i), "world"+strconv.Itoa(i))
	}
	Save(ks, f.Name())

	var err error
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		ks, err = Open(f.Name())
		if err != nil {
			panic(err)
		}
	}
	Save(ks, f.Name())
}

func BenchmarkOpenOldSmall(b *testing.B) {
	f := testFile()
	defer os.Remove(f.Name())
	ks := new(JSONStore)
	for i := 1; i < 10; i++ {
		ks.Set("hello:"+strconv.Itoa(i), "world"+strconv.Itoa(i))
	}
	Save(ks, f.Name())

	var err error
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		ks, err = OpenOld(f.Name())
		if err != nil {
			panic(err)
		}
	}
	Save(ks, f.Name())
}

func BenchmarkOpen(b *testing.B) {
	f := testFile()
	defer os.Remove(f.Name())
	ks := new(JSONStore)
	for i := 1; i < 100; i++ {
		ks.Set("hello:"+strconv.Itoa(i), "world"+strconv.Itoa(i))
	}
	Save(ks, f.Name())

	var err error
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		ks, err = Open(f.Name())
		if err != nil {
			panic(err)
		}
	}
	Save(ks, f.Name())
}

func BenchmarkGet(b *testing.B) {
	ks := new(JSONStore)
	err := ks.Set("human:1", Human{"Dante", 5.4})
	if err != nil {
		panic(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		var human Human
		ks.Get("human:1", &human)
	}
}

func BenchmarkSet(b *testing.B) {
	ks := new(JSONStore)
	b.ResetTimer()
	// set a key to any object you want
	for i := 0; i < b.N; i++ {
		err := ks.Set("human:1", Human{"Dante", 5.4})
		if err != nil {
			panic(err)
		}
	}
}

func BenchmarkSave(b *testing.B) {
	ks := new(JSONStore)
	ks.Set("data", 1234)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		Save(ks, "benchmark.json.gz")
	}
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This binary reads raw conversions from an input file, creates and encrypts partial reports.
// Then send the reports to a HTTP endpoint run by the ad-tech.
package main

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	log "github.com/golang/glog"
	"github.com/hashicorp/go-retryablehttp"
	"github.com/google/privacy-sandbox-aggregation-service/encryption/cryptoio"
	"github.com/google/privacy-sandbox-aggregation-service/pipeline/reporttypes"
	"github.com/google/privacy-sandbox-aggregation-service/test/dpfdataconverter"
	"github.com/google/privacy-sandbox-aggregation-service/test/onepartydataconverter"
	"github.com/google/privacy-sandbox-aggregation-service/utils/utils"
)

// TODO: Store some of the flag values in manifest files.
var (
	address              = flag.String("address", "", "Address of the server.")
	helperPublicKeysURI1 = flag.String("helper_public_keys_uri1", "", "A file that contains the public encryption key from helper1.")
	helperPublicKeysURI2 = flag.String("helper_public_keys_uri2", "", "A file that contains the public encryption key from helper2. Required if helper_origin2 is set.")
	keyBitSize           = flag.Int("key_bit_size", 32, "Bit size of the conversion keys.")
	conversionURI        = flag.String("conversion_uri", "", "Input raw conversion data.")
	conversionRaw        = flag.String("conversion_raw", "2684354560,20", "Raw conversion.")
	sendCount            = flag.Int("send_count", 1, "How many times to send each conversion.")
	helperOrigin1        = flag.String("helper_origin1", "", "Origin of helper1.")
	helperOrigin2        = flag.String("helper_origin2", "", "Origin of helper2. Ignore to use the one-party protocol.")
	concurrency          = flag.Int("concurrency", 10, "Concurrent requests.")

	encryptOutput = flag.Bool("encrypt_output", true, "Generate reports with encryption. This should only be false for integration test before HPKE is ready in Go Tink.")

	impersonatedSvcAccount = flag.String("impersonated_svc_account", "", "Service account to impersonate, skipped if empty")

	version string // set by linker -X
	build   string // set by linker -X
)

func main() {
	flag.Parse()

	buildDate := time.Unix(0, 0)
	if i, err := strconv.ParseInt(build, 10, 64); err != nil {
		log.Error(err)
	} else {
		buildDate = time.Unix(i, 0)
	}

	log.Info("- Debugging enabled - \n")
	log.Infof("Running browser simulator version: %v, build: %v\n", version, buildDate)

	log.Infof("Requests sent to %v", *address)
	log.Infof("Helper public key file locations. 1: %v, 2: %v", *helperPublicKeysURI1, *helperPublicKeysURI2)
	log.Infof("Key Bit size %v", *keyBitSize)
	log.Infof("Conversions file uri: %v", *conversionURI)
	log.Infof("Helper origins. 1: %v, 2: %v", *helperOrigin1, *helperOrigin2)

	client := retryablehttp.NewClient().StandardClient()

	ctx := context.Background()
	var token string
	var err error
	if token, err = utils.GetAuthorizationToken(ctx, *address, *impersonatedSvcAccount); err != nil {
		log.Errorf("Couldn't get Auth Bearer IdToken: %s", err)
	}

	isMPC := *helperOrigin2 != ""

	var conversionsSent uint64
	requestCh := make(chan *bytes.Buffer)
	done := setupRequestWorkers(client, token, *concurrency, &conversionsSent, requestCh)

	var publicKeyInfo1, publicKeyInfo2 []cryptoio.PublicKeyInfo
	// Use any version of the public keys until the version control is designed.
	helperPubKeys1, err := cryptoio.ReadPublicKeyVersions(ctx, *helperPublicKeysURI1)
	if err != nil {
		log.Exit(err)
	}
	for _, v := range helperPubKeys1 {
		publicKeyInfo1 = v
	}
	if isMPC {
		helperPubKeys2, err := cryptoio.ReadPublicKeyVersions(ctx, *helperPublicKeysURI2)
		if err != nil {
			log.Exit(err)
		}
		for _, v := range helperPubKeys2 {
			publicKeyInfo2 = v
		}
	}

	// Empty context information for demo.
	contextInfo, err := utils.MarshalCBOR(&reporttypes.SharedInfo{})
	if err != nil {
		log.Exit(err)
	}

	var conversions []reporttypes.RawReport
	if *conversionURI != "" {
		var err error
		conversions, err = dpfdataconverter.ReadRawConversions(ctx, *conversionURI, *keyBitSize)
		if err != nil {
			log.Exit(err)
		}
	} else {
		conversion, err := dpfdataconverter.ParseRawConversion(*conversionRaw, *keyBitSize)
		if err != nil {
			log.Exit(err)
		}
		conversions = append(conversions, conversion)
	}

	if *sendCount <= 0 {
		*sendCount = 1
	}

	for i := 0; i < *sendCount; i++ {
		for _, c := range conversions {
			var report []byte
			if isMPC {
				key1, key2, err := dpfdataconverter.GenerateDPFKeys(c, *keyBitSize)
				if err != nil {
					log.Exit(err)
				}
				encrypted1, encrypted2, err := dpfdataconverter.EncryptPartialReports(key1, key2, publicKeyInfo1, publicKeyInfo2, contextInfo, *encryptOutput)
				if err != nil {
					log.Exit(err)
				}
				payload1 := &reporttypes.AggregationServicePayload{Origin: *helperOrigin1, Payload: encrypted1.EncryptedReport.Data, KeyID: encrypted2.KeyId}
				payload2 := &reporttypes.AggregationServicePayload{Origin: *helperOrigin2, Payload: encrypted2.EncryptedReport.Data, KeyID: encrypted2.KeyId}
				report, err = utils.MarshalCBOR(&reporttypes.AggregationReport{
					SharedInfo:                 contextInfo,
					AggregationServicePayloads: []*reporttypes.AggregationServicePayload{payload1, payload2},
				})
				if err != nil {
					log.Exit(err)
				}
			} else {
				encrypted, err := onepartydataconverter.EncryptReport(&c, publicKeyInfo1, contextInfo, *encryptOutput)
				if err != nil {
					log.Exit(err)
				}
				payload := &reporttypes.AggregationServicePayload{Origin: *helperOrigin1, Payload: encrypted.EncryptedReport.Data, KeyID: encrypted.KeyId}
				report, err = utils.MarshalCBOR(&reporttypes.AggregationReport{
					SharedInfo:                 contextInfo,
					AggregationServicePayloads: []*reporttypes.AggregationServicePayload{payload},
				})
				if err != nil {
					log.Exit(err)
				}
			}
			requestCh <- bytes.NewBuffer(report)
		}
	}
	close(requestCh)
	<-done
	log.Infof("All %v conversions sent!", conversionsSent)
}

func setupRequestWorkers(client *http.Client, token string, concurrency int, sent *uint64, in <-chan *bytes.Buffer) <-chan bool {
	var wg sync.WaitGroup
	done := make(chan bool)

	worker := func(in <-chan *bytes.Buffer) {
		for data := range in {
			// send request
			req, err := http.NewRequest("POST", *address, data)
			if err != nil {
				log.Error(err)
				continue
			}

			if token != "" {
				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
			}

			req.Header.Set("Content-Type", "encrypted-report")

			resp, err := client.Do(req)
			if err != nil {
				log.Error(err)
				continue
			}
			switch resp.Status {

			case "200 OK":
				atomic.AddUint64(sent, 1)
				log.Infof("%v Conversions sent.", *sent)
			default:
				body, _ := ioutil.ReadAll(resp.Body)
				log.Infof("%v: %s", resp.Status, string(body))
			}
			resp.Body.Close()
		}
		wg.Done()
	}

	for i := 0; i < concurrency; i++ {
		go worker(in)
	}
	wg.Add(concurrency)

	go func() {
		wg.Wait()
		close(done)
	}()
	return done
}
<commit_msg>Fix a key ID bug in the browser simulator.<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This binary reads raw conversions from an input file, creates and encrypts partial reports.
// Then send the reports to a HTTP endpoint run by the ad-tech.
package main

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	log "github.com/golang/glog"
	"github.com/hashicorp/go-retryablehttp"
	"github.com/google/privacy-sandbox-aggregation-service/encryption/cryptoio"
	"github.com/google/privacy-sandbox-aggregation-service/pipeline/reporttypes"
	"github.com/google/privacy-sandbox-aggregation-service/test/dpfdataconverter"
	"github.com/google/privacy-sandbox-aggregation-service/test/onepartydataconverter"
	"github.com/google/privacy-sandbox-aggregation-service/utils/utils"
)

// TODO: Store some of the flag values in manifest files.
var (
	address              = flag.String("address", "", "Address of the server.")
	helperPublicKeysURI1 = flag.String("helper_public_keys_uri1", "", "A file that contains the public encryption key from helper1.")
	helperPublicKeysURI2 = flag.String("helper_public_keys_uri2", "", "A file that contains the public encryption key from helper2. Required if helper_origin2 is set.")
	keyBitSize           = flag.Int("key_bit_size", 32, "Bit size of the conversion keys.")
	conversionURI        = flag.String("conversion_uri", "", "Input raw conversion data.")
	conversionRaw        = flag.String("conversion_raw", "2684354560,20", "Raw conversion.")
	sendCount            = flag.Int("send_count", 1, "How many times to send each conversion.")
	helperOrigin1        = flag.String("helper_origin1", "", "Origin of helper1.")
	helperOrigin2        = flag.String("helper_origin2", "", "Origin of helper2. Ignore to use the one-party protocol.")
	concurrency          = flag.Int("concurrency", 10, "Concurrent requests.")

	encryptOutput = flag.Bool("encrypt_output", true, "Generate reports with encryption. This should only be false for integration test before HPKE is ready in Go Tink.")

	impersonatedSvcAccount = flag.String("impersonated_svc_account", "", "Service account to impersonate, skipped if empty")

	version string // set by linker -X
	build   string // set by linker -X
)

func main() {
	flag.Parse()

	buildDate := time.Unix(0, 0)
	if i, err := strconv.ParseInt(build, 10, 64); err != nil {
		log.Error(err)
	} else {
		buildDate = time.Unix(i, 0)
	}

	log.Info("- Debugging enabled - \n")
	log.Infof("Running browser simulator version: %v, build: %v\n", version, buildDate)

	log.Infof("Requests sent to %v", *address)
	log.Infof("Helper public key file locations. 1: %v, 2: %v", *helperPublicKeysURI1, *helperPublicKeysURI2)
	log.Infof("Key Bit size %v", *keyBitSize)
	log.Infof("Conversions file uri: %v", *conversionURI)
	log.Infof("Helper origins. 1: %v, 2: %v", *helperOrigin1, *helperOrigin2)

	client := retryablehttp.NewClient().StandardClient()

	ctx := context.Background()
	var token string
	var err error
	if token, err = utils.GetAuthorizationToken(ctx, *address, *impersonatedSvcAccount); err != nil {
		log.Errorf("Couldn't get Auth Bearer IdToken: %s", err)
	}

	isMPC := *helperOrigin2 != ""

	var conversionsSent uint64
	requestCh := make(chan *bytes.Buffer)
	done := setupRequestWorkers(client, token, *concurrency, &conversionsSent, requestCh)

	var publicKeyInfo1, publicKeyInfo2 []cryptoio.PublicKeyInfo
	// Use any version of the public keys until the version control is designed.
	helperPubKeys1, err := cryptoio.ReadPublicKeyVersions(ctx, *helperPublicKeysURI1)
	if err != nil {
		log.Exit(err)
	}
	for _, v := range helperPubKeys1 {
		publicKeyInfo1 = v
	}
	if isMPC {
		helperPubKeys2, err := cryptoio.ReadPublicKeyVersions(ctx, *helperPublicKeysURI2)
		if err != nil {
			log.Exit(err)
		}
		for _, v := range helperPubKeys2 {
			publicKeyInfo2 = v
		}
	}

	// Empty context information for demo.
	contextInfo, err := utils.MarshalCBOR(&reporttypes.SharedInfo{})
	if err != nil {
		log.Exit(err)
	}

	var conversions []reporttypes.RawReport
	if *conversionURI != "" {
		var err error
		conversions, err = dpfdataconverter.ReadRawConversions(ctx, *conversionURI, *keyBitSize)
		if err != nil {
			log.Exit(err)
		}
	} else {
		conversion, err := dpfdataconverter.ParseRawConversion(*conversionRaw, *keyBitSize)
		if err != nil {
			log.Exit(err)
		}
		conversions = append(conversions, conversion)
	}

	if *sendCount <= 0 {
		*sendCount = 1
	}

	for i := 0; i < *sendCount; i++ {
		for _, c := range conversions {
			var report []byte
			if isMPC {
				key1, key2, err := dpfdataconverter.GenerateDPFKeys(c, *keyBitSize)
				if err != nil {
					log.Exit(err)
				}
				encrypted1, encrypted2, err := dpfdataconverter.EncryptPartialReports(key1, key2, publicKeyInfo1, publicKeyInfo2, contextInfo, *encryptOutput)
				if err != nil {
					log.Exit(err)
				}
				payload1 := &reporttypes.AggregationServicePayload{Origin: *helperOrigin1, Payload: encrypted1.EncryptedReport.Data, KeyID: encrypted1.KeyId}
				payload2 := &reporttypes.AggregationServicePayload{Origin: *helperOrigin2, Payload: encrypted2.EncryptedReport.Data, KeyID: encrypted2.KeyId}
				report, err = utils.MarshalCBOR(&reporttypes.AggregationReport{
					SharedInfo:                 contextInfo,
					AggregationServicePayloads: []*reporttypes.AggregationServicePayload{payload1, payload2},
				})
				if err != nil {
					log.Exit(err)
				}
			} else {
				encrypted, err := onepartydataconverter.EncryptReport(&c, publicKeyInfo1, contextInfo, *encryptOutput)
				if err != nil {
					log.Exit(err)
				}
				payload := &reporttypes.AggregationServicePayload{Origin: *helperOrigin1, Payload: encrypted.EncryptedReport.Data, KeyID: encrypted.KeyId}
				report, err = utils.MarshalCBOR(&reporttypes.AggregationReport{
					SharedInfo:                 contextInfo,
					AggregationServicePayloads: []*reporttypes.AggregationServicePayload{payload},
				})
				if err != nil {
					log.Exit(err)
				}
			}
			requestCh <- bytes.NewBuffer(report)
		}
	}
	close(requestCh)
	<-done
	log.Infof("All %v conversions sent!", conversionsSent)
}

func setupRequestWorkers(client *http.Client, token string, concurrency int, sent *uint64, in <-chan *bytes.Buffer) <-chan bool {
	var wg sync.WaitGroup
	done := make(chan bool)

	worker := func(in <-chan *bytes.Buffer) {
		for data := range in {
			// send request
			req, err := http.NewRequest("POST", *address, data)
			if err != nil {
				log.Error(err)
				continue
			}

			if token != "" {
				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
			}

			req.Header.Set("Content-Type", "encrypted-report")

			resp, err := client.Do(req)
			if err != nil {
				log.Error(err)
				continue
			}
			switch resp.Status {

			case "200 OK":
				atomic.AddUint64(sent, 1)
				log.Infof("%v Conversions sent.", *sent)
			default:
				body, _ := ioutil.ReadAll(resp.Body)
				log.Infof("%v: %s", resp.Status, string(body))
			}
			resp.Body.Close()
		}
		wg.Done()
	}

	for i := 0; i < concurrency; i++ {
		go worker(in)
	}
	wg.Add(concurrency)

	go func() {
		wg.Wait()
		close(done)
	}()
	return done
}
<|endoftext|>


<commit_before>package sysinfo

// requires sysstat for cpuStats

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"time"
)

const nl = '\n'

// CPUStat contains the information that is generated by mpstat
type CPUStat struct {
	Timestamp string
	CPUID     string
	Usr       int
	Nice      int
	Sys       int
	IOWait    int
	IRQ       int
	Soft      int
	Steal     int
	Guest     int
	GNice     int
	Idle      int
}

func (c CPUStat) String() string {
	return fmt.Sprintf("%s\t%s\t%0.2f\t%0.2f\t%0.2f\t%0.2f\t%0.2f\t%0.2f\t%0.2f\t%0.2f\t%0.2f\t%0.2f\n", c.Timestamp, c.CPUID, float32(c.Usr)/100.0, float32(c.Nice)/100.0, float32(c.Sys)/100.0, float32(c.IOWait)/100.0, float32(c.IRQ)/100.0, float32(c.Soft)/100.0, float32(c.Steal)/100.0, float32(c.Guest)/100.0, float32(c.GNice)/100.0, float32(c.Idle)/100.0)
}

// CPUStats gathers the cpu information for all cores and outputs them over
// the outCh.  The interval is the duration between ticks.  CPU info is
// gathered on each tick.
func CPUStats(interval time.Duration, outCh chan []CPUStat, doneCh chan struct{}) {
	defer close(doneCh)
	var out bytes.Buffer

	for {
		select {
		case <-time.Tick(interval):
			cmd := exec.Command("mpstat", "-P", "ALL")
			cmd.Stdout = &out
			err := cmd.Run()
			if err != nil {
				fmt.Fprintf(os.Stderr, "error getting cpu stats: %s\n", err)
				return
			}
			var x int
			var cpuStats []CPUStat
			// process the output
			for {
				bs, err := out.ReadBytes(nl)
				if err != nil {
					if err == io.EOF {
						break
					}
					fmt.Printf("error reading bytes from command execution: %s\n", err)
					return
				}
				x++
				// skip the first 4 lines
				if x < 4 {
					continue
				}
				// skip empty lines
				if len(bs) == 0 {
					continue
				}
				var cpuStat CPUStat
				cpuStat.Timestamp = string(bs[:11])
				// tmp holds the current field
				tmp := make([]byte, 5)
				// ndx is the counter into tmp
				// fieldNum is used to match up the current value to its struct
				// field; the number does not translate to the ndx
				var ndx, fieldNum, i int
				for _, v := range bs[12:] {
					// 0x20 separates fields, there can be consecutive 0x20
					// occurrences for proper output alignment (when displayed)
					if v == 0x20 {
						// if something has been saved to tmp, it needs to be
						// written to the appropriate field
						if ndx > 0 {
							// CPUID is a string, so don't try to convert to int.
							// convert everything else.
							if fieldNum > 0 {
								fmt.Println(string(tmp[:ndx]))
								i, _ = strconv.Atoi(string(tmp[:ndx]))
							}
							switch fieldNum {
							case 0:
								cpuStat.CPUID = string(tmp[:ndx])
							case 1:
								cpuStat.Usr = i
							case 2:
								cpuStat.Nice = i
							case 3:
								cpuStat.Sys = i
							case 4:
								cpuStat.IOWait = i
							case 5:
								cpuStat.IRQ = i
							case 6:
								cpuStat.Soft = i
							case 7:
								cpuStat.Steal = i
							case 8:
								cpuStat.Guest = i
							case 9:
								cpuStat.GNice = i

							}
							fieldNum++
						}
						// reset for the next field
						ndx = 0
						continue
					}
					// skip . and nl
					if v == 0x2E || v == nl {
						continue
					}
					tmp[ndx] = v
					ndx++
				}
				// the last element hasn't been saved, do it here
				x, err := strconv.Atoi(string(tmp[:ndx]))
				cpuStat.Idle = x
				cpuStats = append(cpuStats, cpuStat)
			}
			// send the current stats
			outCh <- cpuStats
			out.Reset()
		}
	}
}
<commit_msg>CPUStatsTicker for ticker based cpustat gathering and separate out the actual cpu stats gathering into a separate func<commit_after>package sysinfo

// requires sysstat for cpuStats

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"time"
)

const nl = '\n'

// CPUStat contains the information that is generated by mpstat
type CPUStat struct {
	Timestamp string
	CPUID     string
	Usr       int
	Nice      int
	Sys       int
	IOWait    int
	IRQ       int
	Soft      int
	Steal     int
	Guest     int
	GNice     int
	Idle      int
}

func (c CPUStat) String() string {
	return fmt.Sprintf("%s\t%s\t%0.2f\t%0.2f\t%0.2f\t%0.2f\t%0.2f\t%0.2f\t%0.2f\t%0.2f\t%0.2f\t%0.2f\n", c.Timestamp, c.CPUID, float32(c.Usr)/100.0, float32(c.Nice)/100.0, float32(c.Sys)/100.0, float32(c.IOWait)/100.0, float32(c.IRQ)/100.0, float32(c.Soft)/100.0, float32(c.Steal)/100.0, float32(c.Guest)/100.0, float32(c.GNice)/100.0, float32(c.Idle)/100.0)
}

// CPUStatsTicker sends the gathered CPU stats to the outCh.  The interval is
// the duration between ticks.  CPU info is gathered on each tick.
func CPUStatsTicker(interval time.Duration, outCh chan []CPUStat, doneCh chan struct{}) {
	defer close(doneCh)
	for {
		select {
		case <-time.Tick(interval):
			stats, err := CPUStats()
			if err != nil {
				return
			}
			outCh <- stats
		}
	}
}

// CPUStats gathers the CPUStats for all CPUs using mpstat.
func CPUStats() ([]CPUStat, error) {
	var out bytes.Buffer
	cmd := exec.Command("mpstat", "-P", "ALL")
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		fmt.Fprintf(os.Stderr, "error getting cpu stats: %s\n", err)
		return nil, err
	}
	var x int
	var cpuStats []CPUStat
	// process the output
	for {
		bs, err := out.ReadBytes(nl)
		if err != nil {
			if err == io.EOF {
				break
			}
			fmt.Printf("error reading bytes from command execution: %s\n", err)
			return nil, err
		}
		x++
		// skip the first 4 lines
		if x < 4 {
			continue
		}
		// skip empty lines
		if len(bs) == 0 {
			continue
		}
		var cpuStat CPUStat
		cpuStat.Timestamp = string(bs[:11])
		// tmp holds the current field
		tmp := make([]byte, 5)
		// ndx is the counter into tmp
		// fieldNum is used to match up the current value to its struct
		// field; the number does not translate to the ndx
		var ndx, fieldNum, i int
		for _, v := range bs[12:] {
			// 0x20 separates fields, there can be consecutive 0x20
			// occurrences for proper output alignment (when displayed)
			if v == 0x20 {
				// if something has been saved to tmp, it needs to be
				// written to the appropriate field
				if ndx > 0 {
					// CPUID is a string, so don't try to convert to int.
					// convert everything else.
					if fieldNum > 0 {
						fmt.Println(string(tmp[:ndx]))
						i, _ = strconv.Atoi(string(tmp[:ndx]))
					}
					switch fieldNum {
					case 0:
						cpuStat.CPUID = string(tmp[:ndx])
					case 1:
						cpuStat.Usr = i
					case 2:
						cpuStat.Nice = i
					case 3:
						cpuStat.Sys = i
					case 4:
						cpuStat.IOWait = i
					case 5:
						cpuStat.IRQ = i
					case 6:
						cpuStat.Soft = i
					case 7:
						cpuStat.Steal = i
					case 8:
						cpuStat.Guest = i
					case 9:
						cpuStat.GNice = i

					}
					fieldNum++
				}
				// reset for the next field
				ndx = 0
				continue
			}
			// skip . and nl
			if v == 0x2E || v == nl {
				continue
			}
			tmp[ndx] = v
			ndx++
		}
		// the last element hasn't been saved, do it here
		x, err := strconv.Atoi(string(tmp[:ndx]))
		cpuStat.Idle = x
		cpuStats = append(cpuStats, cpuStat)
	}
	return cpuStats, nil
}
<|endoftext|>


<commit_before>package radar

import (
	"errors"
	"os"
	"time"

	"github.com/concourse/atc"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/resource"
	"github.com/concourse/atc/worker"
	"github.com/tedsuo/ifrit"

	"github.com/pivotal-golang/lager"
)

var errResourceNoLongerConfigured = errors.New("resource no longer configured")

//go:generate counterfeiter . RadarDB

type RadarDB interface {
	GetPipelineName() string
	ScopedName(string) string

	IsPaused() (bool, error)

	GetConfig() (atc.Config, db.ConfigVersion, error)

	GetLatestVersionedResource(resource db.SavedResource) (db.SavedVersionedResource, error)
	GetResource(resourceName string) (db.SavedResource, error)
	PauseResource(resourceName string) error
	UnpauseResource(resourceName string) error

	SaveResourceVersions(atc.ResourceConfig, []atc.Version) error
	SetResourceCheckError(resource db.SavedResource, err error) error
}

type Radar struct {
	logger lager.Logger

	tracker resource.Tracker

	interval time.Duration

	locker Locker
	db     RadarDB
}

func NewRadar(
	tracker resource.Tracker,
	interval time.Duration,
	locker Locker,
	db RadarDB,
) *Radar {
	return &Radar{
		tracker:  tracker,
		interval: interval,
		locker:   locker,
		db:       db,
	}
}

func (radar *Radar) Scanner(logger lager.Logger, resourceName string) ifrit.Runner {
	return ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		ticker := time.NewTicker(radar.interval)

		close(ready)

		for {
			select {
			case <-signals:
				return nil

			case <-ticker.C:
				lock := radar.checkLock(radar.db.ScopedName(resourceName))
				resourceCheckingLock, err := radar.locker.AcquireWriteLockImmediately(lock)

				if err != nil {
					continue
				}

				err = radar.scan(logger.Session("tick"), resourceName)

				resourceCheckingLock.Release()

				if err != nil {
					return err
				}
			}
		}
	})
}

func (radar *Radar) Scan(logger lager.Logger, resourceName string) error {
	lock, err := radar.locker.AcquireWriteLock(radar.checkLock(radar.db.ScopedName(resourceName)))
	if err != nil {
		return err
	}

	defer lock.Release()

	return radar.scan(logger, resourceName)
}

func (radar *Radar) scan(logger lager.Logger, resourceName string) error {
	pipelinePaused, err := radar.db.IsPaused()
	if err != nil {
		logger.Error("failed-to-check-if-pipeline-paused", err)
		return err
	}

	if pipelinePaused {
		logger.Debug("pipeline-paused")
		return nil
	}

	config, _, err := radar.db.GetConfig()
	if err != nil {
		logger.Error("failed-to-get-config", err)
		// don't propagate error; we can just retry next tick
		return nil
	}

	resourceConfig, found := config.Resources.Lookup(resourceName)
	if !found {
		logger.Info("resource-removed-from-configuration")
		// return an error so that we exit
		return errResourceNoLongerConfigured
	}

	savedResource, err := radar.db.GetResource(resourceName)
	if err != nil {
		return err
	}

	if savedResource.Paused {
		return nil
	}

	typ := resource.ResourceType(resourceConfig.Type)

	res, err := radar.tracker.Init(checkIdentifier(radar.db.GetPipelineName(), resourceConfig), typ)
	if err != nil {
		logger.Error("failed-to-initialize-new-resource", err)
		return err
	}

	defer res.Release()

	var from db.Version
	if vr, err := radar.db.GetLatestVersionedResource(savedResource); err == nil {
		from = vr.Version
	}

	logger.Debug("checking", lager.Data{
		"from": from,
	})

	newVersions, err := res.Check(resourceConfig.Source, atc.Version(from))
	setErr := radar.db.SetResourceCheckError(savedResource, err)
	if setErr != nil {
		logger.Error("failed-to-set-check-error", err)
	}

	if err != nil {
		logger.Error("failed-to-check", err)

		return err
	}

	if len(newVersions) == 0 {
		logger.Debug("no-new-versions")
		return nil
	}

	logger.Info("versions-found", lager.Data{
		"versions": newVersions,
		"total":    len(newVersions),
	})

	err = radar.db.SaveResourceVersions(resourceConfig, newVersions)
	if err != nil {
		logger.Error("failed-to-save-versions", err, lager.Data{
			"versions": newVersions,
		})
	}

	return nil
}

func (radar *Radar) checkLock(resourceName string) []db.NamedLock {
	return []db.NamedLock{db.ResourceCheckingLock(resourceName)}
}

func checkIdentifier(pipelineName string, res atc.ResourceConfig) resource.Session {
	return resource.Session{
		ID: worker.Identifier{
			PipelineName: pipelineName,

			Name: res.Name,
			Type: "check",

			CheckType:   res.Type,
			CheckSource: res.Source,
		},
		Ephemeral: true,
	}
}
<commit_msg>Add a nicer error for missing resources<commit_after>package radar

import (
	"fmt"
	"os"
	"time"

	"github.com/concourse/atc"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/resource"
	"github.com/concourse/atc/worker"
	"github.com/tedsuo/ifrit"

	"github.com/pivotal-golang/lager"
)

type resourceNotConfiguredError struct {
	ResourceName string
}

func (err resourceNotConfiguredError) Error() string {
	return fmt.Sprintf("resource '%s' was not found in config", err.ResourceName)
}

//go:generate counterfeiter . RadarDB

type RadarDB interface {
	GetPipelineName() string
	ScopedName(string) string

	IsPaused() (bool, error)

	GetConfig() (atc.Config, db.ConfigVersion, error)

	GetLatestVersionedResource(resource db.SavedResource) (db.SavedVersionedResource, error)
	GetResource(resourceName string) (db.SavedResource, error)
	PauseResource(resourceName string) error
	UnpauseResource(resourceName string) error

	SaveResourceVersions(atc.ResourceConfig, []atc.Version) error
	SetResourceCheckError(resource db.SavedResource, err error) error
}

type Radar struct {
	logger lager.Logger

	tracker resource.Tracker

	interval time.Duration

	locker Locker
	db     RadarDB
}

func NewRadar(
	tracker resource.Tracker,
	interval time.Duration,
	locker Locker,
	db RadarDB,
) *Radar {
	return &Radar{
		tracker:  tracker,
		interval: interval,
		locker:   locker,
		db:       db,
	}
}

func (radar *Radar) Scanner(logger lager.Logger, resourceName string) ifrit.Runner {
	return ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		ticker := time.NewTicker(radar.interval)

		close(ready)

		for {
			select {
			case <-signals:
				return nil

			case <-ticker.C:
				lock := radar.checkLock(radar.db.ScopedName(resourceName))
				resourceCheckingLock, err := radar.locker.AcquireWriteLockImmediately(lock)

				if err != nil {
					continue
				}

				err = radar.scan(logger.Session("tick"), resourceName)

				resourceCheckingLock.Release()

				if err != nil {
					return err
				}
			}
		}
	})
}

func (radar *Radar) Scan(logger lager.Logger, resourceName string) error {
	lock, err := radar.locker.AcquireWriteLock(radar.checkLock(radar.db.ScopedName(resourceName)))
	if err != nil {
		return err
	}

	defer lock.Release()

	return radar.scan(logger, resourceName)
}

func (radar *Radar) scan(logger lager.Logger, resourceName string) error {
	pipelinePaused, err := radar.db.IsPaused()
	if err != nil {
		logger.Error("failed-to-check-if-pipeline-paused", err)
		return err
	}

	if pipelinePaused {
		logger.Debug("pipeline-paused")
		return nil
	}

	config, _, err := radar.db.GetConfig()
	if err != nil {
		logger.Error("failed-to-get-config", err)
		// don't propagate error; we can just retry next tick
		return nil
	}

	resourceConfig, found := config.Resources.Lookup(resourceName)
	if !found {
		logger.Info("resource-removed-from-configuration")
		// return an error so that we exit
		return resourceNotConfiguredError{ResourceName: resourceName}
	}

	savedResource, err := radar.db.GetResource(resourceName)
	if err != nil {
		return err
	}

	if savedResource.Paused {
		return nil
	}

	typ := resource.ResourceType(resourceConfig.Type)

	res, err := radar.tracker.Init(checkIdentifier(radar.db.GetPipelineName(), resourceConfig), typ)
	if err != nil {
		logger.Error("failed-to-initialize-new-resource", err)
		return err
	}

	defer res.Release()

	var from db.Version
	if vr, err := radar.db.GetLatestVersionedResource(savedResource); err == nil {
		from = vr.Version
	}

	logger.Debug("checking", lager.Data{
		"from": from,
	})

	newVersions, err := res.Check(resourceConfig.Source, atc.Version(from))
	setErr := radar.db.SetResourceCheckError(savedResource, err)
	if setErr != nil {
		logger.Error("failed-to-set-check-error", err)
	}

	if err != nil {
		logger.Error("failed-to-check", err)

		return err
	}

	if len(newVersions) == 0 {
		logger.Debug("no-new-versions")
		return nil
	}

	logger.Info("versions-found", lager.Data{
		"versions": newVersions,
		"total":    len(newVersions),
	})

	err = radar.db.SaveResourceVersions(resourceConfig, newVersions)
	if err != nil {
		logger.Error("failed-to-save-versions", err, lager.Data{
			"versions": newVersions,
		})
	}

	return nil
}

func (radar *Radar) checkLock(resourceName string) []db.NamedLock {
	return []db.NamedLock{db.ResourceCheckingLock(resourceName)}
}

func checkIdentifier(pipelineName string, res atc.ResourceConfig) resource.Session {
	return resource.Session{
		ID: worker.Identifier{
			PipelineName: pipelineName,

			Name: res.Name,
			Type: "check",

			CheckType:   res.Type,
			CheckSource: res.Source,
		},
		Ephemeral: true,
	}
}
<|endoftext|>


<commit_before>package cemi

import (
	"errors"
	"io"

	"github.com/vapourismo/knx-go/knx/encoding"
)

// A LData is a link-layer data frame.
type LData struct {
	Control1    ControlField1
	Control2    ControlField2
	Source      uint16
	Destination uint16
	Data        TPDU
}

// ReadFrom initializes the LData structure by reading from the given Reader.
func (ldata *LData) ReadFrom(r io.Reader) (n int64, err error) {
	var tpduLen8 uint8
	len, err := encoding.ReadSome(
		r,
		&ldata.Control1,
		&ldata.Control2,
		&ldata.Source,
		&ldata.Destination,
		&tpduLen8,
	)
	n += len

	if err != nil {
		return n, err
	}

	tpdu := make([]byte, int(tpduLen8)+1)
	len, err = encoding.Read(r, tpdu)
	n += len

	if err != nil {
		return n, err
	}

	ldata.Data = TPDU(tpdu)

	return
}

// WriteTo serializes the LData structure and writes it to the given Writer.
func (ldata *LData) WriteTo(w io.Writer) (int64, error) {
	if len(ldata.Data) < 1 {
		return 0, errors.New("TPDU length has be 1 or more")
	} else if len(ldata.Data) > 256 {
		return 0, errors.New("TPDU is too large")
	}

	return encoding.WriteSome(
		w,
		ldata.Control1,
		ldata.Control2,
		ldata.Source,
		ldata.Destination,
		byte(len(ldata.Data)-1),
		ldata.Data,
	)
}
<commit_msg>Simplify LData ReadFrom<commit_after>package cemi

import (
	"errors"
	"io"

	"github.com/vapourismo/knx-go/knx/encoding"
)

// A LData is a link-layer data frame. Represents L_Data.req, L_Data.con and L_Data.ind since they
// all share the same structure.
type LData struct {
	Control1    ControlField1
	Control2    ControlField2
	Source      uint16
	Destination uint16
	Data        TPDU
}

// ReadFrom initializes the LData structure by reading from the given Reader.
func (ldata *LData) ReadFrom(r io.Reader) (n int64, err error) {
	var tpduLen8 uint8
	n, err = encoding.ReadSome(
		r,
		&ldata.Control1,
		&ldata.Control2,
		&ldata.Source,
		&ldata.Destination,
		&tpduLen8,
	)

	if err != nil {
		return
	}

	tpdu := make([]byte, int(tpduLen8)+1)
	m, err := encoding.Read(r, tpdu)
	n += m

	if err != nil {
		return
	}

	ldata.Data = TPDU(tpdu)

	return
}

// WriteTo serializes the LData structure and writes it to the given Writer.
func (ldata *LData) WriteTo(w io.Writer) (int64, error) {
	if len(ldata.Data) < 1 {
		return 0, errors.New("TPDU length has be 1 or more")
	} else if len(ldata.Data) > 256 {
		return 0, errors.New("TPDU is too large")
	}

	return encoding.WriteSome(
		w,
		ldata.Control1,
		ldata.Control2,
		ldata.Source,
		ldata.Destination,
		byte(len(ldata.Data)-1),
		ldata.Data,
	)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"go/build"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"testing"
)

func Test_newCommand(t *testing.T) {
	cmd := &newCommand{}
	for _, v := range [][]interface{}{
		[]interface{}{"Name", cmd.Name(), "new"},
		[]interface{}{"Alias", cmd.Alias(), ""},
		[]interface{}{"Short", cmd.Short(), "create a new application"},
		[]interface{}{"Usage", cmd.Usage(), "new APP_PATH"},
	} {
		name, actual, expected := v[0], v[1], v[2]
		if !reflect.DeepEqual(actual, expected) {
			t.Errorf(".%v expect %v, but %v", name, expected, actual)
		}
	}
	if cmd.flag != nil {
		t.Fatalf("Expect nil, but %v", cmd.flag)
	}
	flags := flag.NewFlagSet("testflags", flag.ExitOnError)
	cmd.DefineFlags(flags)
	flags.Parse([]string{})
	actual, expected := cmd.flag, flags
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("Expect %v, but %v", expected, actual)
	}
}

func Test_newCommand_Run_with_no_APP_PATH_given(t *testing.T) {
	cmd := &newCommand{}
	flags := flag.NewFlagSet("testflags", flag.ExitOnError)
	cmd.DefineFlags(flags)
	flags.Parse([]string{})
	defer func() {
		if err := recover(); err == nil {
			t.Errorf("Expect panic, but not occurred")
		}
	}()
	cmd.Run()
}

func Test_newCommand_Run_with_already_exists(t *testing.T) {
	tempdir, err := ioutil.TempDir("", "Test_newRun_with_already_exists")
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(tempdir)
	appPath := filepath.Base(tempdir)
	dstPath := filepath.Join(tempdir, "src", appPath)
	configDir := filepath.Join(dstPath, "config")
	if err := os.MkdirAll(configDir, 0755); err != nil {
		panic(err)
	}
	if err := ioutil.WriteFile(filepath.Join(configDir, "app.go"), nil, 0644); err != nil {
		panic(err)
	}
	cmd := &newCommand{}
	flags := flag.NewFlagSet("testflags", flag.ExitOnError)
	cmd.DefineFlags(flags)
	flags.Parse([]string{appPath})
	defer func() {
		if err := recover(); err == nil {
			t.Errorf("Expect panic, but not occurred")
		}
	}()
	origGOPATH := build.Default.GOPATH
	defer func() {
		build.Default.GOPATH = origGOPATH
	}()
	build.Default.GOPATH = tempdir + string(filepath.ListSeparator) + build.Default.GOPATH
	cmd.Run()
}

func Test_newCommand_Run(t *testing.T) {
	tempdir, err := ioutil.TempDir("", "Test_newRun")
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(tempdir)
	appPath := filepath.Base(tempdir)
	dstPath := filepath.Join(tempdir, "src", appPath)
	f, err := os.OpenFile(os.DevNull, os.O_WRONLY, os.ModePerm)
	if err != nil {
		panic(err)
	}
	oldStdout, oldStderr := os.Stdout, os.Stderr
	os.Stdout, os.Stderr = f, f
	defer func() {
		os.Stdout, os.Stderr = oldStdout, oldStderr
	}()
	cmd := &newCommand{}
	flags := flag.NewFlagSet("testflags", flag.ExitOnError)
	cmd.DefineFlags(flags)
	flags.Parse([]string{appPath})
	origGOPATH := build.Default.GOPATH
	defer func() {
		build.Default.GOPATH = origGOPATH
	}()
	build.Default.GOPATH = tempdir + string(filepath.ListSeparator) + build.Default.GOPATH
	cmd.Run()
	var actuals []string
	filepath.Walk(dstPath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			panic(err)
		}
		if info.IsDir() {
			return nil
		}
		actuals = append(actuals, path)
		return nil
	})
	expects := []string{
		filepath.Join("app", "controllers", "root.go"),
		filepath.Join("app", "views", "layouts", "app.html"),
		filepath.Join("app", "views", "root.html"),
		filepath.Join("config", "app.go"),
		filepath.Join("config", "routes.go"),
		filepath.Join("config", "dev", "app.go"),
		filepath.Join("config", "prod", "app.go"),
		filepath.Join("dev.go"),
		filepath.Join("prod.go"),
	}
	sort.Strings(actuals)
	sort.Strings(expects)
	for i, _ := range actuals {
		actual := actuals[i]
		expected := filepath.Join(dstPath, expects[i])
		if actual != expected {
			t.Errorf("Expect %v, but %v", expected, actual)
		}
	}
}
<commit_msg>Fix broken tests<commit_after>package main

import (
	"flag"
	"go/build"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"testing"
)

func Test_newCommand(t *testing.T) {
	cmd := &newCommand{}
	for _, v := range [][]interface{}{
		[]interface{}{"Name", cmd.Name(), "new"},
		[]interface{}{"Alias", cmd.Alias(), ""},
		[]interface{}{"Short", cmd.Short(), "create a new application"},
		[]interface{}{"Usage", cmd.Usage(), "new APP_PATH"},
	} {
		name, actual, expected := v[0], v[1], v[2]
		if !reflect.DeepEqual(actual, expected) {
			t.Errorf(".%v expect %v, but %v", name, expected, actual)
		}
	}
	if cmd.flag != nil {
		t.Fatalf("Expect nil, but %v", cmd.flag)
	}
	flags := flag.NewFlagSet("testflags", flag.ExitOnError)
	cmd.DefineFlags(flags)
	flags.Parse([]string{})
	actual, expected := cmd.flag, flags
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("Expect %v, but %v", expected, actual)
	}
}

func Test_newCommand_Run_with_no_APP_PATH_given(t *testing.T) {
	cmd := &newCommand{}
	flags := flag.NewFlagSet("testflags", flag.ExitOnError)
	cmd.DefineFlags(flags)
	flags.Parse([]string{})
	defer func() {
		if err := recover(); err == nil {
			t.Errorf("Expect panic, but not occurred")
		}
	}()
	cmd.Run()
}

func Test_newCommand_Run_with_already_exists(t *testing.T) {
	tempdir, err := ioutil.TempDir("", "Test_newRun_with_already_exists")
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(tempdir)
	appPath := filepath.Base(tempdir)
	dstPath := filepath.Join(tempdir, "src", appPath)
	configDir := filepath.Join(dstPath, "config")
	if err := os.MkdirAll(configDir, 0755); err != nil {
		panic(err)
	}
	if err := ioutil.WriteFile(filepath.Join(configDir, "app.go"), nil, 0644); err != nil {
		panic(err)
	}
	cmd := &newCommand{}
	flags := flag.NewFlagSet("testflags", flag.ExitOnError)
	cmd.DefineFlags(flags)
	flags.Parse([]string{appPath})
	defer func() {
		if err := recover(); err == nil {
			t.Errorf("Expect panic, but not occurred")
		}
	}()
	origGOPATH := build.Default.GOPATH
	defer func() {
		build.Default.GOPATH = origGOPATH
	}()
	build.Default.GOPATH = tempdir + string(filepath.ListSeparator) + build.Default.GOPATH
	cmd.Run()
}

func Test_newCommand_Run(t *testing.T) {
	tempdir, err := ioutil.TempDir("", "Test_newRun")
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(tempdir)
	appPath := filepath.Base(tempdir)
	dstPath := filepath.Join(tempdir, "src", appPath)
	f, err := os.OpenFile(os.DevNull, os.O_WRONLY, os.ModePerm)
	if err != nil {
		panic(err)
	}
	oldStdout, oldStderr := os.Stdout, os.Stderr
	os.Stdout, os.Stderr = f, f
	defer func() {
		os.Stdout, os.Stderr = oldStdout, oldStderr
	}()
	cmd := &newCommand{}
	flags := flag.NewFlagSet("testflags", flag.ExitOnError)
	cmd.DefineFlags(flags)
	flags.Parse([]string{appPath})
	origGOPATH := build.Default.GOPATH
	defer func() {
		build.Default.GOPATH = origGOPATH
	}()
	build.Default.GOPATH = tempdir + string(filepath.ListSeparator) + build.Default.GOPATH
	cmd.Run()
	var actuals []string
	filepath.Walk(dstPath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			panic(err)
		}
		if info.IsDir() {
			return nil
		}
		actuals = append(actuals, path)
		return nil
	})
	expects := []string{
		filepath.Join("app", "controllers", "root.go"),
		filepath.Join("app", "views", "layouts", "app.html"),
		filepath.Join("app", "views", "root.html"),
		filepath.Join("config", "app.go"),
		filepath.Join("config", "routes.go"),
		filepath.Join("config", "dev", "app.go"),
		filepath.Join("config", "prod", "app.go"),
		filepath.Join("dev.go"),
		filepath.Join("prod.go"),
		filepath.Join("public", "robots.txt"),
	}
	sort.Strings(actuals)
	sort.Strings(expects)
	for i, _ := range actuals {
		actual := actuals[i]
		expected := filepath.Join(dstPath, expects[i])
		if actual != expected {
			t.Errorf("Expect %v, but %v", expected, actual)
		}
	}
}
<|endoftext|>


<commit_before>package l10n

import (
	"errors"
	"fmt"

	"github.com/jinzhu/gorm"
)

func BeforeQuery(scope *gorm.Scope) {
	if isLocalizable(scope) {
		if locale, ok := getLocale(scope); ok { // is locale
			quotedTableName := scope.QuotedTableName()
			switch mode, _ := scope.DB().Get("l10n:mode"); mode {
			case "locale":
				scope.Search.Where(fmt.Sprintf("%v.language_code = ?", quotedTableName), locale)
			case "global":
				scope.Search.Where(fmt.Sprintf("%v.language_code = ?", quotedTableName), Global)
			default:
				scope.Search.Where(fmt.Sprintf("%v.language_code = ? OR %v.language_code = ?", quotedTableName, quotedTableName), locale, Global).Order("language_code DESC")
			}
		}
	}
}

func BeforeCreate(scope *gorm.Scope) {
	if isLocalizable(scope) {
		if locale, ok := getLocale(scope); ok { // is locale
			if isLocaleCreateable(scope) || !scope.PrimaryKeyZero() {
				setLocale(scope, locale)
			} else {
				scope.Err(errors.New("permission denied to create from locale"))
			}
		} else {
			setLocale(scope, Global)
		}
	}
}

func BeforeUpdate(scope *gorm.Scope) {
	if isLocalizable(scope) {
		if locale, ok := getLocale(scope); ok { // is locale
			scope.Search.Unscoped = true
			scope.Search.Where(fmt.Sprintf("%v.language_code = ?", scope.QuotedTableName()), locale)
			setLocale(scope, locale)
			scope.Search.Omit(syncColumns(scope)...)
		} else {
			setLocale(scope, Global)
		}
	}
}

func AfterUpdate(scope *gorm.Scope) {
	if isLocalizable(scope) {
		if locale, ok := getLocale(scope); ok {
			if scope.DB().RowsAffected == 0 { //is locale and nothing updated
				var count int
				var query = fmt.Sprintf("language_code = ? AND %v = ?", scope.PrimaryKey())
				if scope.NewDB().Table(scope.TableName()).Where(query, locale, scope.PrimaryKeyValue()).Count(&count); count == 0 {
					scope.DB().Create(scope.Value)
				}
			}
		} else if syncColumns := syncColumns(scope); len(syncColumns) > 0 { // is global
			scope.NewDB().Where(fmt.Sprintf("%v = ?"), scope.PrimaryKeyValue).Select(syncColumns).Update(scope.Value)
		}
	}
}

func BeforeDelete(scope *gorm.Scope) {
	if isLocalizable(scope) {
		if locale, ok := getLocale(scope); ok { // is locale
			scope.Search.Where("language_code = ?", locale)
		}
	}
}

func RegisterCallbacks(db *gorm.DB) {
	callback := db.Callback()

	callback.Create().Before("gorm:before_create").Register("l10n:before_create", BeforeCreate)

	callback.Update().Before("gorm:before_update").Register("l10n:before_update", BeforeUpdate)
	callback.Update().After("gorm:after_update").Register("l10n:after_update", AfterUpdate)

	callback.Delete().Before("gorm:before_delete").Register("l10n:before_delete", BeforeDelete)

	callback.Query().Before("gorm:query").Register("l10n:before_query", BeforeQuery)
}
<commit_msg>Tweak l10n display logic for admin<commit_after>package l10n

import (
	"errors"
	"fmt"

	"github.com/jinzhu/gorm"
)

func BeforeQuery(scope *gorm.Scope) {
	if isLocalizable(scope) {
		quotedTableName := scope.QuotedTableName()
		if locale, ok := getLocale(scope); ok { // is locale
			switch mode, _ := scope.DB().Get("l10n:mode"); mode {
			case "locale":
				scope.Search.Where(fmt.Sprintf("%v.language_code = ?", quotedTableName), locale)
			case "global":
				scope.Search.Where(fmt.Sprintf("%v.language_code = ?", quotedTableName), Global)
			default:
				primaryKey := scope.PrimaryKey()
				joinCondition := fmt.Sprintf("LEFT JOIN %v t2 ON t2.%v = %v.%v AND t2.language_code = ?", quotedTableName, primaryKey, quotedTableName, primaryKey)
				scope.Search.Joins(joinCondition).Where(fmt.Sprintf("%v.language_code = ?", quotedTableName), locale, Global)
			}
		} else {
			scope.Search.Where(fmt.Sprintf("%v.language_code = ?", quotedTableName), Global)
		}
	}
}

func BeforeCreate(scope *gorm.Scope) {
	if isLocalizable(scope) {
		if locale, ok := getLocale(scope); ok { // is locale
			if isLocaleCreateable(scope) || !scope.PrimaryKeyZero() {
				setLocale(scope, locale)
			} else {
				scope.Err(errors.New("permission denied to create from locale"))
			}
		} else {
			setLocale(scope, Global)
		}
	}
}

func BeforeUpdate(scope *gorm.Scope) {
	if isLocalizable(scope) {
		if locale, ok := getLocale(scope); ok { // is locale
			scope.Search.Unscoped = true
			scope.Search.Where(fmt.Sprintf("%v.language_code = ?", scope.QuotedTableName()), locale)
			setLocale(scope, locale)
			scope.Search.Omit(syncColumns(scope)...)
		} else {
			setLocale(scope, Global)
		}
	}
}

func AfterUpdate(scope *gorm.Scope) {
	if isLocalizable(scope) {
		if locale, ok := getLocale(scope); ok {
			if scope.DB().RowsAffected == 0 { //is locale and nothing updated
				var count int
				var query = fmt.Sprintf("language_code = ? AND %v = ?", scope.PrimaryKey())
				if scope.NewDB().Table(scope.TableName()).Where(query, locale, scope.PrimaryKeyValue()).Count(&count); count == 0 {
					scope.DB().Create(scope.Value)
				}
			}
		} else if syncColumns := syncColumns(scope); len(syncColumns) > 0 { // is global
			scope.NewDB().Where(fmt.Sprintf("%v = ?"), scope.PrimaryKeyValue).Select(syncColumns).Update(scope.Value)
		}
	}
}

func BeforeDelete(scope *gorm.Scope) {
	if isLocalizable(scope) {
		if locale, ok := getLocale(scope); ok { // is locale
			scope.Search.Where("language_code = ?", locale)
		}
	}
}

func RegisterCallbacks(db *gorm.DB) {
	callback := db.Callback()

	callback.Create().Before("gorm:before_create").Register("l10n:before_create", BeforeCreate)

	callback.Update().Before("gorm:before_update").Register("l10n:before_update", BeforeUpdate)
	callback.Update().After("gorm:after_update").Register("l10n:after_update", AfterUpdate)

	callback.Delete().Before("gorm:before_delete").Register("l10n:before_delete", BeforeDelete)

	callback.Query().Before("gorm:query").Register("l10n:before_query", BeforeQuery)
}
<|endoftext|>


<commit_before>/* {{{ Copyright (c) Paul R. Tagliamonte <paultag@debian.org>, 2015
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE. }}} */

package sbuild

import (
	"fmt"
	"os/exec"
)

type Sbuild struct {
	args []string
}

func (sbuild Sbuild) Partial(args ...string) Sbuild {
	sbuild.args = append(sbuild.args, args...)
	return sbuild
}

func (sbuild *Sbuild) AddFlag(flag string) {
	sbuild.args = append(sbuild.args, flag)
}

func (sbuild *Sbuild) AddArgument(key, value string) {
	sbuild.args = append(sbuild.args, fmt.Sprintf("--%s=%s", key, value))
}

func (sbuild *Sbuild) BuildCommand(dsc string) (*exec.Cmd, error) {
	s := sbuild.Partial(dsc)
	cmd := exec.Command("sbuild", s.args...)
	return cmd, nil
}

func (sbuild *Sbuild) Build(dsc string) (*exec.Cmd, error) {
	cmd, err := sbuild.BuildCommand(dsc)
	if err != nil {
		return nil, err
	}
	return cmd, cmd.Start()
}

func NewSbuild(chroot string, suite string, args ...string) Sbuild {
	sbuild := Sbuild{
		args: append([]string{"-c", chroot, "-d", suite}, args...),
	}
	return sbuild
}

// vim: foldmethod=marker
<commit_msg>deal with paramless thingers.<commit_after>/* {{{ Copyright (c) Paul R. Tagliamonte <paultag@debian.org>, 2015
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE. }}} */

package sbuild

import (
	"fmt"
	"os/exec"
)

type Sbuild struct {
	args []string
}

func (sbuild Sbuild) Partial(args ...string) Sbuild {
	sbuild.args = append(sbuild.args, args...)
	return sbuild
}

func (sbuild *Sbuild) AddFlag(flag string) {
	sbuild.args = append(sbuild.args, flag)
}

func (sbuild *Sbuild) AddArgument(key string, value *string) {
	if value == nil {
		return sbuild.AddFlag("--" + key)
	}
	sbuild.args = append(sbuild.args, fmt.Sprintf("--%s=%s", key, *value))
}

func (sbuild *Sbuild) BuildCommand(dsc string) (*exec.Cmd, error) {
	s := sbuild.Partial(dsc)
	cmd := exec.Command("sbuild", s.args...)
	return cmd, nil
}

func (sbuild *Sbuild) Build(dsc string) (*exec.Cmd, error) {
	cmd, err := sbuild.BuildCommand(dsc)
	if err != nil {
		return nil, err
	}
	return cmd, cmd.Start()
}

func NewSbuild(chroot string, suite string, args ...string) Sbuild {
	sbuild := Sbuild{
		args: append([]string{"-c", chroot, "-d", suite}, args...),
	}
	return sbuild
}

// vim: foldmethod=marker
<|endoftext|>