<commit_before>package masterapi

import (
	"encoding/json"
	"fmt"
	"github.com/KIT-MAMID/mamid/model"
	"github.com/gorilla/mux"
	"github.com/jinzhu/gorm"
	"github.com/mattn/go-sqlite3"
	"net/http"
	"strconv"
)

type Slave struct {
	ID                   int64  `json:"id"`
	Hostname             string `json:"hostname"`
	Port                 uint   `json:"slave_port"`
	MongodPortRangeBegin uint   `json:"mongod_port_range_begin"` //inclusive
	MongodPortRangeEnd   uint   `json:"mongod_port_range_end"`   //exclusive
	PersistentStorage    bool   `json:"persistent_storage"`
	ConfiguredState      string `json:"configured_state"`
	RiskGroupID          *int64 `json:"risk_group_id"`
}

func (m *MasterAPI) SlaveIndex(w http.ResponseWriter, r *http.Request) {
	tx := m.DB.Begin()
	defer tx.Rollback()

	var slaves []*model.Slave
	err := tx.Order("id", false).Find(&slaves).Error
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		return
	}

	out := make([]*Slave, len(slaves))
	for i, v := range slaves {
		out[i] = ProjectModelSlaveToSlave(v)
	}
	json.NewEncoder(w).Encode(out)
}

func (m *MasterAPI) SlaveById(w http.ResponseWriter, r *http.Request) {
	idStr := mux.Vars(r)["slaveId"]
	id64, err := strconv.ParseUint(idStr, 10, 0)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	id := uint(id64)

	if id == 0 {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "id may not be 0")
		return
	}

	tx := m.DB.Begin()
	defer tx.Rollback()

	var slave model.Slave
	res := tx.First(&slave, id)

	if res.RecordNotFound() {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	if err = res.Error; err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		return
	}

	json.NewEncoder(w).Encode(ProjectModelSlaveToSlave(&slave))
	return
}

func (m *MasterAPI) SlavePut(w http.ResponseWriter, r *http.Request) {
	var postSlave Slave
	err := json.NewDecoder(r.Body).Decode(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "cannot parse object (%s)", err.Error())
		return
	}

	// Validation

	if postSlave.ID != 0 {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "must not specify the slave ID in PUT request")
		return
	}

	modelSlave, err := ProjectSlaveToModelSlave(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, err.Error())
		return
	}

	// Persist to database

	tx := m.DB.Begin()

	err = tx.Create(&modelSlave).Error

	//Check db specific errors
	if driverErr, ok := err.(sqlite3.Error); ok && driverErr.ExtendedCode == sqlite3.ErrConstraintUnique {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, driverErr.Error())
		tx.Rollback()
		return
	}

	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		tx.Rollback()
		return
	}

	// Trigger cluster allocator
	if err = m.attemptClusterAllocator(tx, w); err != nil {
		return
	}

	tx.Commit()

	// Return created slave

	json.NewEncoder(w).Encode(ProjectModelSlaveToSlave(modelSlave))

	return
}

func (m *MasterAPI) SlaveUpdate(w http.ResponseWriter, r *http.Request) {
	idStr := mux.Vars(r)["slaveId"]
	id, err := strconv.ParseInt(idStr, 10, 0)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	var postSlave Slave
	err = json.NewDecoder(r.Body).Decode(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "cannot parse object (%s)", err.Error())
		return
	}

	// Validation

	if postSlave.ID != id {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "must not change the id of an object")
		return
	}

	if err = postSlave.assertNoZeroFieldsSet(); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "must not POST JSON with zero values in any field: %s", err.Error())
		return
	}

	// Only allow changes to both observed and desired disabled slaves

	tx := m.DB.Begin()

	var modelSlave model.Slave
	modelSlaveRes := tx.First(&modelSlave, id)
	if modelSlaveRes.RecordNotFound() {
		w.WriteHeader(http.StatusNotFound)
		return
	} else if err = modelSlaveRes.Error; err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		tx.Rollback()
		return
	}

	updatedModelSlave, err := ProjectSlaveToModelSlave(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, err)
		return
	}

	// Only allow changes to both observed and desired disabled slaves

	permissionError, dbError := changeToSlaveAllowed(tx, &modelSlave, updatedModelSlave)
	if dbError != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, dbError)
		tx.Rollback()
		return
	}
	if permissionError != nil {
		w.WriteHeader(http.StatusForbidden)
		fmt.Fprint(w, permissionError)
		tx.Rollback()
		return
	}

	// Persist to database
	err = tx.Save(&updatedModelSlave).Error

	//Check db specific errors
	if driverErr, ok := err.(sqlite3.Error); ok {
		if driverErr.ExtendedCode == sqlite3.ErrConstraintUnique {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprint(w, driverErr.Error())
			tx.Rollback()
			return
		}
	}

	// Trigger cluster allocator
	if err = m.attemptClusterAllocator(tx, w); err != nil {
		return
	}

	tx.Commit()

}

func (m *MasterAPI) SlaveDelete(w http.ResponseWriter, r *http.Request) {
	idStr := mux.Vars(r)["slaveId"]
	id, err := strconv.ParseInt(idStr, 10, 0)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	tx := m.DB.Begin()

	// Can only delete disabled slaves
	var currentSlave model.Slave
	if err = tx.First(&currentSlave, id).Related(&currentSlave.Mongods, "Mongods").Error; err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		tx.Rollback()
		return
	}

	if len(currentSlave.Mongods) != 0 {
		w.WriteHeader(http.StatusForbidden)
		fmt.Fprintf(w, "slave with id %d has active Mongods", currentSlave.ID)
		tx.Rollback()
		return
	}

	// Allow delete

	s := tx.Delete(&model.Slave{ID: id})
	if s.Error != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, s.Error.Error())
		tx.Rollback()
		return
	}

	if s.RowsAffected > 1 {
		masterapiLog.Errorf("inconsistency: slave DELETE affected more than one row. Slave.ID = %v", id)
	}

	if s.RowsAffected == 0 {
		w.WriteHeader(http.StatusNotFound)
		tx.Rollback()
		return
	}

	// Trigger cluster allocator
	if err = m.attemptClusterAllocator(tx, w); err != nil {
		return
	}

	tx.Commit()
}

func changeToSlaveAllowed(tx *gorm.DB, currentSlave *model.Slave, updatedSlave *model.Slave) (permissionError, dbError error) {

	//Allow change of state if nothing else is changed
	if currentSlave.ID == updatedSlave.ID &&
		currentSlave.Hostname == updatedSlave.Hostname &&
		currentSlave.Port == updatedSlave.Port &&
		currentSlave.MongodPortRangeBegin == updatedSlave.MongodPortRangeBegin &&
		currentSlave.MongodPortRangeEnd == updatedSlave.MongodPortRangeEnd &&
		currentSlave.PersistentStorage == updatedSlave.PersistentStorage &&
		currentSlave.RiskGroupID == updatedSlave.RiskGroupID {
		return nil, nil
	}
	if currentSlave.ConfiguredState != model.SlaveStateDisabled && currentSlave.ConfiguredState != model.SlaveStateMaintenance {
		return fmt.Errorf("slave's desired state must be `maintenance` or `disabled`"), nil
	}

	if err := tx.Model(&currentSlave).Related(&currentSlave.Mongods, "Mongods").Error; err != nil {
		return nil, err
	}

	if len(currentSlave.Mongods) != 0 {
		//The port range of a slave with mongods may not be reduced because the mongods may be using the ports
		if updatedSlave.MongodPortRangeBegin > currentSlave.MongodPortRangeBegin || updatedSlave.MongodPortRangeEnd < currentSlave.MongodPortRangeEnd {
			return fmt.Errorf("The port range may not be reduced since the slave still has Mongods"), nil
		}

		//The persistence of slaves with mongods may not be changed because the mongods depend on the persistence
		if updatedSlave.PersistentStorage != currentSlave.PersistentStorage {
			return fmt.Errorf("The persistence may not be changed since the slave still has Mongods"), nil
		}
	}

	return nil, nil

}
<commit_msg>ADD: masterapi: extend comment for acceptance of slave.configured_state change<commit_after>package masterapi

import (
	"encoding/json"
	"fmt"
	"github.com/KIT-MAMID/mamid/model"
	"github.com/gorilla/mux"
	"github.com/jinzhu/gorm"
	"github.com/mattn/go-sqlite3"
	"net/http"
	"strconv"
)

type Slave struct {
	ID                   int64  `json:"id"`
	Hostname             string `json:"hostname"`
	Port                 uint   `json:"slave_port"`
	MongodPortRangeBegin uint   `json:"mongod_port_range_begin"` //inclusive
	MongodPortRangeEnd   uint   `json:"mongod_port_range_end"`   //exclusive
	PersistentStorage    bool   `json:"persistent_storage"`
	ConfiguredState      string `json:"configured_state"`
	RiskGroupID          *int64 `json:"risk_group_id"`
}

func (m *MasterAPI) SlaveIndex(w http.ResponseWriter, r *http.Request) {
	tx := m.DB.Begin()
	defer tx.Rollback()

	var slaves []*model.Slave
	err := tx.Order("id", false).Find(&slaves).Error
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		return
	}

	out := make([]*Slave, len(slaves))
	for i, v := range slaves {
		out[i] = ProjectModelSlaveToSlave(v)
	}
	json.NewEncoder(w).Encode(out)
}

func (m *MasterAPI) SlaveById(w http.ResponseWriter, r *http.Request) {
	idStr := mux.Vars(r)["slaveId"]
	id64, err := strconv.ParseUint(idStr, 10, 0)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	id := uint(id64)

	if id == 0 {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "id may not be 0")
		return
	}

	tx := m.DB.Begin()
	defer tx.Rollback()

	var slave model.Slave
	res := tx.First(&slave, id)

	if res.RecordNotFound() {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	if err = res.Error; err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		return
	}

	json.NewEncoder(w).Encode(ProjectModelSlaveToSlave(&slave))
	return
}

func (m *MasterAPI) SlavePut(w http.ResponseWriter, r *http.Request) {
	var postSlave Slave
	err := json.NewDecoder(r.Body).Decode(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "cannot parse object (%s)", err.Error())
		return
	}

	// Validation

	if postSlave.ID != 0 {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "must not specify the slave ID in PUT request")
		return
	}

	modelSlave, err := ProjectSlaveToModelSlave(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, err.Error())
		return
	}

	// Persist to database

	tx := m.DB.Begin()

	err = tx.Create(&modelSlave).Error

	//Check db specific errors
	if driverErr, ok := err.(sqlite3.Error); ok && driverErr.ExtendedCode == sqlite3.ErrConstraintUnique {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, driverErr.Error())
		tx.Rollback()
		return
	}

	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		tx.Rollback()
		return
	}

	// Trigger cluster allocator
	if err = m.attemptClusterAllocator(tx, w); err != nil {
		return
	}

	tx.Commit()

	// Return created slave

	json.NewEncoder(w).Encode(ProjectModelSlaveToSlave(modelSlave))

	return
}

func (m *MasterAPI) SlaveUpdate(w http.ResponseWriter, r *http.Request) {
	idStr := mux.Vars(r)["slaveId"]
	id, err := strconv.ParseInt(idStr, 10, 0)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	var postSlave Slave
	err = json.NewDecoder(r.Body).Decode(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "cannot parse object (%s)", err.Error())
		return
	}

	// Validation

	if postSlave.ID != id {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "must not change the id of an object")
		return
	}

	if err = postSlave.assertNoZeroFieldsSet(); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "must not POST JSON with zero values in any field: %s", err.Error())
		return
	}

	// Only allow changes to both observed and desired disabled slaves

	tx := m.DB.Begin()

	var modelSlave model.Slave
	modelSlaveRes := tx.First(&modelSlave, id)
	if modelSlaveRes.RecordNotFound() {
		w.WriteHeader(http.StatusNotFound)
		return
	} else if err = modelSlaveRes.Error; err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		tx.Rollback()
		return
	}

	updatedModelSlave, err := ProjectSlaveToModelSlave(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, err)
		return
	}

	// Only allow changes to both observed and desired disabled slaves

	permissionError, dbError := changeToSlaveAllowed(tx, &modelSlave, updatedModelSlave)
	if dbError != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, dbError)
		tx.Rollback()
		return
	}
	if permissionError != nil {
		w.WriteHeader(http.StatusForbidden)
		fmt.Fprint(w, permissionError)
		tx.Rollback()
		return
	}

	// Persist to database
	err = tx.Save(&updatedModelSlave).Error

	//Check db specific errors
	if driverErr, ok := err.(sqlite3.Error); ok {
		if driverErr.ExtendedCode == sqlite3.ErrConstraintUnique {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprint(w, driverErr.Error())
			tx.Rollback()
			return
		}
	}

	// Trigger cluster allocator
	if err = m.attemptClusterAllocator(tx, w); err != nil {
		return
	}

	tx.Commit()

}

func (m *MasterAPI) SlaveDelete(w http.ResponseWriter, r *http.Request) {
	idStr := mux.Vars(r)["slaveId"]
	id, err := strconv.ParseInt(idStr, 10, 0)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	tx := m.DB.Begin()

	// Can only delete disabled slaves
	var currentSlave model.Slave
	if err = tx.First(&currentSlave, id).Related(&currentSlave.Mongods, "Mongods").Error; err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		tx.Rollback()
		return
	}

	if len(currentSlave.Mongods) != 0 {
		w.WriteHeader(http.StatusForbidden)
		fmt.Fprintf(w, "slave with id %d has active Mongods", currentSlave.ID)
		tx.Rollback()
		return
	}

	// Allow delete

	s := tx.Delete(&model.Slave{ID: id})
	if s.Error != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, s.Error.Error())
		tx.Rollback()
		return
	}

	if s.RowsAffected > 1 {
		masterapiLog.Errorf("inconsistency: slave DELETE affected more than one row. Slave.ID = %v", id)
	}

	if s.RowsAffected == 0 {
		w.WriteHeader(http.StatusNotFound)
		tx.Rollback()
		return
	}

	// Trigger cluster allocator
	if err = m.attemptClusterAllocator(tx, w); err != nil {
		return
	}

	tx.Commit()
}

func changeToSlaveAllowed(tx *gorm.DB, currentSlave *model.Slave, updatedSlave *model.Slave) (permissionError, dbError error) {

	// Allow change of state if nothing else is changed
	// NOTE: changing the slave state is an indication to the ClusterAllocator but has no direct consequences in deployment
	if currentSlave.ID == updatedSlave.ID &&
		currentSlave.Hostname == updatedSlave.Hostname &&
		currentSlave.Port == updatedSlave.Port &&
		currentSlave.MongodPortRangeBegin == updatedSlave.MongodPortRangeBegin &&
		currentSlave.MongodPortRangeEnd == updatedSlave.MongodPortRangeEnd &&
		currentSlave.PersistentStorage == updatedSlave.PersistentStorage &&
		currentSlave.RiskGroupID == updatedSlave.RiskGroupID {
		return nil, nil
	}
	if currentSlave.ConfiguredState != model.SlaveStateDisabled && currentSlave.ConfiguredState != model.SlaveStateMaintenance {
		return fmt.Errorf("slave's desired state must be `maintenance` or `disabled`"), nil
	}

	if err := tx.Model(&currentSlave).Related(&currentSlave.Mongods, "Mongods").Error; err != nil {
		return nil, err
	}

	if len(currentSlave.Mongods) != 0 {
		//The port range of a slave with mongods may not be reduced because the mongods may be using the ports
		if updatedSlave.MongodPortRangeBegin > currentSlave.MongodPortRangeBegin || updatedSlave.MongodPortRangeEnd < currentSlave.MongodPortRangeEnd {
			return fmt.Errorf("The port range may not be reduced since the slave still has Mongods"), nil
		}

		//The persistence of slaves with mongods may not be changed because the mongods depend on the persistence
		if updatedSlave.PersistentStorage != currentSlave.PersistentStorage {
			return fmt.Errorf("The persistence may not be changed since the slave still has Mongods"), nil
		}
	}

	return nil, nil

}
<|endoftext|>


<commit_before>package pggameday

import (
	"database/sql"
	"github.com/ecopony/gamedayapi"
	_ "github.com/lib/pq" // PostgreSQL driver
	"log"
)

// ImportPitchesForTeamAndYears saves all pitch data fields for a team and season.
func ImportPitchesForTeamAndYears(teamCode string, years []int) {
	log.Println("Importing for " + teamCode)

	// Assumes a pg database exists named go-gameday, a role that can access it.
	// Assumes a table called pitches with a character column called code.
	db, err := sql.Open("postgres", "user=go-gameday dbname=go-gameday sslmode=disable")
	issue := db.Ping()
	log.Println(issue)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	db.Exec("DROP TABLE IF EXISTS pitches")
	db.Exec(`CREATE TABLE pitches (pitchid SERIAL PRIMARY KEY, game_id varchar(40), year int, inning int, half varchar(6),
	at_bat_num int, at_bat_b int, at_bat_s int, at_bat_o int, at_bat_start_tfs int, batter int, stand char(1), b_height
	varchar(4), pitcher int, p_throws char(1), at_bat_des varchar(400), at_bat_event varchar(20), pitch_des varchar(40),
	pitch_id int, pitch_type char(2), type_confidence DECIMAL(4, 3), pitch_tfs int, pitch_x DECIMAL(5, 2), pitch_y
	DECIMAL(5, 2), pitch_sv_id varchar(40), pitch_start_speed DECIMAL(4, 1), pitch_end_speed DECIMAL(4, 1), sz_top
	DECIMAL(3, 2), sz_bottom DECIMAL(3, 2), pfx_x DECIMAL(4, 2), pfx_z DECIMAL(4, 2), px DECIMAL(4, 3), pz
	DECIMAL(4, 3), x0 DECIMAL(5, 3), y0 DECIMAL(5, 3), z0 DECIMAL(5, 3), vx0 DECIMAL(4, 2), vy0 DECIMAL(6, 3), vz0
	DECIMAL(5, 3), ax DECIMAL(5, 3), ay DECIMAL(5, 3), az DECIMAL(5, 3), break_y DECIMAL(3, 1), break_angle
	DECIMAL(4, 1), break_length DECIMAL(3, 1), zone int, spin_dir DECIMAL(6, 3), spin_rate DECIMAL(7, 3))`)

	fetchFunction := func(game *gamedayapi.Game) {
		log.Println(">>>> " + game.ID + " <<<<")
		for _, inning := range game.AllInnings().Innings {
			half := "top"
			for _, atBat := range inning.AtBats() {
				if atBat.O == "3" {
					half = "bottom"
				}
				for _, pitch := range atBat.Pitches {
					log.Println("> " + atBat.StartTFS)

					res, err := db.Query(`INSERT INTO pitches
						(game_id, year, inning, half, at_bat_num, at_bat_b, at_bat_s, at_bat_o, at_bat_start_tfs,
						batter, stand, b_height, pitcher, p_throws, at_bat_des, at_bat_event,
						pitch_des, pitch_id, pitch_type, type_confidence, pitch_tfs
						)
						VALUES
						($1, $2, $3, $4, $5, $6, $7, $8, $9,
						$10, $11, $12, $13, $14, $15, $16,
						$17, $18, $19, $20, $21)`,
						game.ID, game.Year(), inning.Num, half, atBat.Num, atBat.B, atBat.S, atBat.O, nullableString(atBat.StartTFS),
						atBat.Batter, atBat.Stand, atBat.BHeight, atBat.Pitcher, atBat.PThrows, atBat.Des, atBat.Event,
						pitch.Des, pitch.ID, pitch.Type, nullableString(pitch.TypeConfidence), nullableString(pitch.TFS))
					if err != nil {
						log.Fatal(err)
					}
					res.Close()
				}
			}
		}
	}
	gamedayapi.FetchByTeamAndYears(teamCode, years, fetchFunction)
}

func nullableString(value string) interface {} {
	if value == "" {
		return nil
	}
	return value
}
<commit_msg>Working through the rest of the pitch attributes on the pg import.<commit_after>package pggameday

import (
	"database/sql"
	"github.com/ecopony/gamedayapi"
	_ "github.com/lib/pq" // PostgreSQL driver
	"log"
)

// ImportPitchesForTeamAndYears saves all pitch data fields for a team and season.
func ImportPitchesForTeamAndYears(teamCode string, years []int) {
	log.Println("Importing for " + teamCode)

	// Assumes a pg database exists named go-gameday, a role that can access it.
	// Assumes a table called pitches with a character column called code.
	db, err := sql.Open("postgres", "user=go-gameday dbname=go-gameday sslmode=disable")
	issue := db.Ping()
	log.Println(issue)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	db.Exec("DROP TABLE IF EXISTS pitches")
	db.Exec(`CREATE TABLE pitches (pitchid SERIAL PRIMARY KEY, game_id varchar(40), year int, inning int, half varchar(6),
	at_bat_num int, at_bat_b int, at_bat_s int, at_bat_o int, at_bat_start_tfs int, batter int, stand char(1), b_height
	varchar(4), pitcher int, p_throws char(1), at_bat_des varchar(400), at_bat_event varchar(20), pitch_des varchar(40),
	pitch_id int, pitch_type char(2), type_confidence DECIMAL(4, 3), pitch_tfs int, pitch_x DECIMAL(5, 2), pitch_y
	DECIMAL(5, 2), pitch_sv_id varchar(40), pitch_start_speed DECIMAL(4, 1), pitch_end_speed DECIMAL(4, 1), sz_top
	DECIMAL(3, 2), sz_bottom DECIMAL(3, 2), pfx_x DECIMAL(4, 2), pfx_z DECIMAL(4, 2), px DECIMAL(4, 3), pz
	DECIMAL(4, 3), x0 DECIMAL(5, 3), y0 DECIMAL(5, 3), z0 DECIMAL(5, 3), vx0 DECIMAL(4, 2), vy0 DECIMAL(6, 3), vz0
	DECIMAL(5, 3), ax DECIMAL(5, 3), ay DECIMAL(5, 3), az DECIMAL(5, 3), break_y DECIMAL(3, 1), break_angle
	DECIMAL(4, 1), break_length DECIMAL(3, 1), zone int, spin_dir DECIMAL(6, 3), spin_rate DECIMAL(7, 3))`)

	fetchFunction := func(game *gamedayapi.Game) {
		log.Println(">>>> " + game.ID + " <<<<")
		for _, inning := range game.AllInnings().Innings {
			half := "top"
			for _, atBat := range inning.AtBats() {
				if atBat.O == "3" {
					half = "bottom"
				}
				for _, pitch := range atBat.Pitches {
					// log.Println("> " + pitch.SzTop)

					res, err := db.Query(`INSERT INTO pitches
						(game_id, year, inning, half, at_bat_num, at_bat_b, at_bat_s, at_bat_o, at_bat_start_tfs,
						batter, stand, b_height, pitcher, p_throws, at_bat_des, at_bat_event,
						pitch_des, pitch_id, pitch_type, type_confidence, pitch_tfs,
						pitch_x, pitch_y, pitch_sv_id, pitch_start_speed, pitch_end_speed,
						sz_top, sz_bottom, pfx_x, pfx_z,
						px, pz, x0, y0,
						z0, vx0, vy0, vz0,
						ax, ay, az, break_y,
						break_angle, break_length, zone,
						spin_dir, spin_rate
						)
						VALUES
						($1, $2, $3, $4, $5, $6, $7, $8, $9,
						$10, $11, $12, $13, $14, $15, $16,
						$17, $18, $19, $20, $21,
						$22, $23, $24, $25, $26,
						$27, $28, $29, $30,
						$31, $32, $33, $34,
						$35, $36, $37, $38,
						$39, $40, $41, $42,
						$43, $44, $45,
						$46, $47)`,
						game.ID, game.Year(), inning.Num, half, atBat.Num, atBat.B, atBat.S, atBat.O, nullableString(atBat.StartTFS),
						atBat.Batter, atBat.Stand, atBat.BHeight, atBat.Pitcher, atBat.PThrows, atBat.Des, atBat.Event,
						pitch.Des, pitch.ID, pitch.Type, nullableString(pitch.TypeConfidence), nullableString(pitch.TFS),
						pitch.X, pitch.Y, pitch.SvID, nullableString(pitch.StartSpeed), nullableString(pitch.EndSpeed),
						nullableString(pitch.SzTop), nullableString(pitch.SzBottom), nullableString(pitch.PFXX), nullableString(pitch.PFXZ),
						nullableString(pitch.PX), nullableString(pitch.PZ), nullableString(pitch.X0), nullableString(pitch.Y0),
						nullableString(pitch.Z0), nullableString(pitch.VX0), nullableString(pitch.VY0), nullableString(pitch.VZ0),
						nullableString(pitch.AX), nullableString(pitch.AY), nullableString(pitch.AZ), nullableString(pitch.BreakY),
						nullableString(pitch.BreakAngle), nullableString(pitch.BreakLength), nullableString(pitch.Zone),
						nullableString(pitch.SpinDir), nullableString(pitch.SpinRate))
					if err != nil {
						log.Fatal(err)
					}
					res.Close()
				}
			}
		}
	}
	gamedayapi.FetchByTeamAndYears(teamCode, years, fetchFunction)
}

func nullableString(value string) interface{} {
	if value == "" {
		return nil
	}
	return value
}
<|endoftext|>


<commit_before>package gui

import (
	"github.com/felixangell/nate/gfx"
	"github.com/veandco/go-sdl2/sdl"
	"github.com/veandco/go-sdl2/sdl_ttf"
	"github.com/vinzmay/go-rope"
	"unicode/utf8"
)

const (
	TAB_SIZE int32 = 4
)

type Cursor struct {
	x, y   int
	rx, ry int
}

func (c *Cursor) move(x, y int) {
	c.move_render(x, y, x, y)
}

// moves the cursors position, and the
// rendered coordinates by the given amount
func (c *Cursor) move_render(x, y, rx, ry int) {
	c.x += x
	c.y += y

	c.rx += rx
	c.ry += ry
}

const (
	cursor_flash_ms uint32 = 400
	reset_delay_ms  uint32 = 400
)

var (
	should_draw  bool   = false
	should_flash bool   = true
	timer        uint32 = 0
	reset_timer  uint32 = 0
)

type Buffer struct {
	ComponentLocation
	font          *ttf.Font
	contents      []*rope.Rope
	curs          *Cursor
	input_handler *InputHandler
}

func NewBuffer() *Buffer {
	font, err := ttf.OpenFont("./res/firacode.ttf", 24)
	if err != nil {
		panic(err)
	}

	buff := &Buffer{
		contents: []*rope.Rope{},
		font:     font,
		curs:     &Cursor{},
	}
	buff.appendLine("This is a test.")
	return buff
}

func (b *Buffer) Dispose() {
	for _, texture := range TEXTURE_CACHE {
		texture.Destroy()
	}
}

func (b *Buffer) Init() {}

func (b *Buffer) GetComponents() []Component {
	return []Component{}
}

func (b *Buffer) AddComponent(c Component) {}

func (b *Buffer) SetInputHandler(i *InputHandler) {
	b.input_handler = i
}

func (b *Buffer) GetInputHandler() *InputHandler {
	return b.input_handler
}

func (b *Buffer) appendLine(val string) {
	b.contents = append(b.contents, rope.New(val))
	b.curs.move(len(val), 0)
}

func (b *Buffer) processTextInput(t *sdl.TextInputEvent) {
	// TODO: how the fuck do decode this properly?
	raw_val, size := utf8.DecodeLastRune(t.Text[:1])
	if raw_val == utf8.RuneError || size == 0 {
		return
	}

	b.contents[b.curs.y] = b.contents[b.curs.y].Insert(b.curs.x, string(raw_val))
	b.curs.move(1, 0)
}

func (b *Buffer) processActionKey(t *sdl.KeyDownEvent) {
	switch t.Keysym.Scancode {
	case sdl.SCANCODE_RETURN:
		initial_x := b.curs.x
		prev_line_len := b.contents[b.curs.y].Len()

		var new_rope *rope.Rope
		if initial_x < prev_line_len && initial_x > 0 {
			left, right := b.contents[b.curs.y].Split(initial_x)
			new_rope = right
			b.contents[b.curs.y] = left
		} else if initial_x == 0 {
			b.contents = append(b.contents, new(rope.Rope))      // grow
			copy(b.contents[b.curs.y+1:], b.contents[b.curs.y:]) // shift
			b.contents[b.curs.y] = new(rope.Rope)                // set
			b.curs.move(0, 1)
			return
		} else {
			new_rope = rope.New(" ")
		}

		b.curs.move(0, 1)
		for x := 0; x < initial_x; x++ {
			b.curs.move(-1, 0)
		}
		b.contents = append(b.contents, new_rope)
	case sdl.SCANCODE_BACKSPACE:
		if b.curs.x > 0 {
			b.contents[b.curs.y] = b.contents[b.curs.y].Delete(b.curs.x, 1)
			b.curs.move(-1, 0)
		} else if b.curs.x == 0 && b.curs.y > 0 {
			// start of line, wrap to previous
			// two cases here:

			// the line_len is zero, in which case
			// we delete the line and go to the end
			// of the previous line
			if b.contents[b.curs.y].Len() == 0 {
				b.curs.move(b.contents[b.curs.y-1].Len(), -1)
				// FIXME, delete from the curs.y dont pop!
				b.contents = b.contents[:len(b.contents)-1]
				return
			}

			// or, the line has characters, so we join
			// that line with the previous line
			prev_line_len := b.contents[b.curs.y-1].Len()
			b.contents[b.curs.y-1] = b.contents[b.curs.y-1].Concat(b.contents[b.curs.y])
			b.curs.move(prev_line_len, -1)

			// FIXME delete from curs.y, not pop!
			b.contents = b.contents[:len(b.contents)-1]
		}
	case sdl.SCANCODE_RIGHT:
		curr_line_length := b.contents[b.curs.y].Len()
		if b.curs.x >= curr_line_length && b.curs.y < len(b.contents)-1 {
			// we're at the end of the line and we have
			// some lines after, let's wrap around
			b.curs.move(0, 1)
			b.curs.move(-curr_line_length, 0)
		} else if b.curs.x < b.contents[b.curs.y].Len() {
			// we have characters to the right, let's move along
			b.curs.move(1, 0)
		}
	case sdl.SCANCODE_LEFT:
		if b.curs.x == 0 && b.curs.y > 0 {
			b.curs.move(b.contents[b.curs.y-1].Len(), -1)

		} else if b.curs.x > 0 {
			b.curs.move(-1, 0)
		}
	case sdl.SCANCODE_TAB:
		// TODO
	}
}

func renderString(font *ttf.Font, val string, col sdl.Color, smooth bool) *sdl.Surface {
	if smooth {
		text, err := font.RenderUTF8_Blended(val, col)
		if err != nil {
			panic(err)
		}
		return text
	} else {
		text, err := font.RenderUTF8_Solid(val, col)
		if err != nil {
			panic(err)
		}
		return text
	}
	return nil
}

func (b *Buffer) Translate(x, y int32) {
	b.x += x
	b.y += y
}

func (b *Buffer) Update() {
	prev_x := b.curs.x
	prev_y := b.curs.y

	if b.input_handler == nil {
		panic("help")
	}

	if b.input_handler.Event != nil {
		switch t := b.input_handler.Event.(type) {
		case *sdl.TextInputEvent:
			b.processTextInput(t)
		case *sdl.KeyDownEvent:
			b.processActionKey(t)
		}
	}

	if b.curs.x != prev_x || b.curs.y != prev_y {
		should_draw = true
		should_flash = false
		reset_timer = sdl.GetTicks()
	}

	if !should_flash && sdl.GetTicks()-reset_timer > reset_delay_ms {
		should_flash = true
	}

	if sdl.GetTicks()-timer > cursor_flash_ms && should_flash {
		timer = sdl.GetTicks()
		should_draw = !should_draw
	}
}

var last_w, last_h int32
var TEXTURE_CACHE map[rune]*sdl.Texture = map[rune]*sdl.Texture{}

func (b *Buffer) Render(ctx *sdl.Renderer) {

	// render the ol' cursor
	if should_draw {
		gfx.SetDrawColorHex(ctx, 0x657B83)
		ctx.FillRect(&sdl.Rect{
			b.x + (int32(b.curs.rx)+1)*last_w,
			b.y + int32(b.curs.ry)*last_h,
			last_w,
			last_h,
		})
	}

	var y_col int32
	for _, rope := range b.contents {
		// this is because if we had the following
		// text input:
		//
		// Foo
		// _			<-- underscore is a space!
		// Blah
		// and we delete that underscore... it causes
		// a panic because there are no characters in
		// the empty string!
		if rope.Len() == 0 {
			// even though the string is empty
			// we still need to offset it by a line
			y_col += 1
			continue
		}

		var x_col int32
		for _, char := range rope.String() {
			switch char {
			case '\n':
				x_col = 0
				y_col += 1
				continue
			case '\t':
				x_col += TAB_SIZE
				continue
			}

			x_col += 1

			text := renderString(b.font, string(char), gfx.HexColor(0x7a7a7a), true)
			defer text.Free()

			last_w = text.W
			last_h = text.H

			texture, ok := TEXTURE_CACHE[char]
			if !ok {
				// can't find it in the cache so we
				// load and then cache it.
				texture, _ = ctx.CreateTextureFromSurface(text)
				TEXTURE_CACHE[char] = texture
			}

			// FIXME still kinda slow
			// we can also cull so that
			// we don't render things that aren't
			// visible outside of the component
			ctx.Copy(texture, nil, &sdl.Rect{
				b.x + (x_col * text.W),
				b.y + (y_col * text.H),
				text.W,
				text.H,
			})
		}

		y_col += 1
	}
}
<commit_msg>Tab indentation with \t and spaces. Closes #2<commit_after>package gui

import (
	"github.com/felixangell/nate/gfx"
	"github.com/veandco/go-sdl2/sdl"
	"github.com/veandco/go-sdl2/sdl_ttf"
	"github.com/vinzmay/go-rope"
	"unicode/utf8"
)

type Cursor struct {
	x, y   int
	rx, ry int
}

func (c *Cursor) move(x, y int) {
	c.move_render(x, y, x, y)
}

// moves the cursors position, and the
// rendered coordinates by the given amount
func (c *Cursor) move_render(x, y, rx, ry int) {
	c.x += x
	c.y += y

	c.rx += rx
	c.ry += ry
}

var (
	cursor_flash_ms uint32 = 400
	reset_delay_ms  uint32 = 400

	// buffer
	TAB_SIZE int32 = 4
	HUNGRY_BACKSPACE bool = true
	TABS_ARE_SPACES bool = true

	// cursor
	should_draw  bool   = false
	should_flash bool   = true
	timer        uint32 = 0
	reset_timer  uint32 = 0
)

type Buffer struct {
	ComponentLocation
	font          *ttf.Font
	contents      []*rope.Rope
	curs          *Cursor
	input_handler *InputHandler
}

func NewBuffer() *Buffer {
	font, err := ttf.OpenFont("./res/firacode.ttf", 24)
	if err != nil {
		panic(err)
	}

	buff := &Buffer{
		contents: []*rope.Rope{},
		font:     font,
		curs:     &Cursor{},
	}
	buff.appendLine("This is a test.")
	return buff
}

func (b *Buffer) Dispose() {
	for _, texture := range TEXTURE_CACHE {
		texture.Destroy()
	}
}

func (b *Buffer) Init() {}

func (b *Buffer) GetComponents() []Component {
	return []Component{}
}

func (b *Buffer) AddComponent(c Component) {}

func (b *Buffer) SetInputHandler(i *InputHandler) {
	b.input_handler = i
}

func (b *Buffer) GetInputHandler() *InputHandler {
	return b.input_handler
}

func (b *Buffer) appendLine(val string) {
	b.contents = append(b.contents, rope.New(val))
	b.curs.move(len(val), 0)
}

func (b *Buffer) processTextInput(t *sdl.TextInputEvent) {
	// TODO: how the fuck do decode this properly?
	raw_val, size := utf8.DecodeLastRune(t.Text[:1])
	if raw_val == utf8.RuneError || size == 0 {
		return
	}

	b.contents[b.curs.y] = b.contents[b.curs.y].Insert(b.curs.x, string(raw_val))
	b.curs.move(1, 0)
}

func (b *Buffer) processActionKey(t *sdl.KeyDownEvent) {
	switch t.Keysym.Scancode {
	case sdl.SCANCODE_RETURN:
		initial_x := b.curs.x
		prev_line_len := b.contents[b.curs.y].Len()

		var new_rope *rope.Rope
		if initial_x < prev_line_len && initial_x > 0 {
			left, right := b.contents[b.curs.y].Split(initial_x)
			new_rope = right
			b.contents[b.curs.y] = left
		} else if initial_x == 0 {
			b.contents = append(b.contents, new(rope.Rope))      // grow
			copy(b.contents[b.curs.y+1:], b.contents[b.curs.y:]) // shift
			b.contents[b.curs.y] = new(rope.Rope)                // set
			b.curs.move(0, 1)
			return
		} else {
			new_rope = rope.New(" ")
		}

		b.curs.move(0, 1)
		for x := 0; x < initial_x; x++ {
			// TODO(Felix): there's a bug here where
			// this doesn't account for the rendered x
			// position when we use tabs as tabs and not spaces
			b.curs.move(-1, 0)
		}
		b.contents = append(b.contents, new_rope)
	case sdl.SCANCODE_BACKSPACE:
		if b.curs.x > 0 {
			offs := -1
			if !TABS_ARE_SPACES {
				if b.contents[b.curs.y].Index(b.curs.x) == '\t' {
					offs = int(-TAB_SIZE)
				}
			} else if HUNGRY_BACKSPACE && b.curs.x >= int(TAB_SIZE) && TABS_ARE_SPACES {
				// why x + 1 here? wtf
				if b.contents[b.curs.y].Substr((b.curs.x + 1) - int(TAB_SIZE), int(TAB_SIZE)).String() == "    " {
					// delete {TAB_SIZE} amount of characters
					// from the cursors x pos
					for i := 0; i < int(TAB_SIZE); i++ {
						b.contents[b.curs.y] = b.contents[b.curs.y].Delete(b.curs.x, 1)
						b.curs.move(-1, 0)
					}
					break
				}
			}

			b.contents[b.curs.y] = b.contents[b.curs.y].Delete(b.curs.x, 1)
			b.curs.move_render(-1, 0, offs, 0)
		} else if b.curs.x == 0 && b.curs.y > 0 {
			// start of line, wrap to previous
			// two cases here:

			// the line_len is zero, in which case
			// we delete the line and go to the end
			// of the previous line
			if b.contents[b.curs.y].Len() == 0 {
				b.curs.move(b.contents[b.curs.y-1].Len(), -1)
				// FIXME, delete from the curs.y dont pop!
				b.contents = b.contents[:len(b.contents)-1]
				return
			}

			// or, the line has characters, so we join
			// that line with the previous line
			prev_line_len := b.contents[b.curs.y-1].Len()
			b.contents[b.curs.y-1] = b.contents[b.curs.y-1].Concat(b.contents[b.curs.y])
			b.curs.move(prev_line_len, -1)

			// FIXME delete from curs.y, not pop!
			b.contents = b.contents[:len(b.contents)-1]
		}
	case sdl.SCANCODE_RIGHT:
		curr_line_length := b.contents[b.curs.y].Len()
		if b.curs.x >= curr_line_length && b.curs.y < len(b.contents)-1 {
			// we're at the end of the line and we have
			// some lines after, let's wrap around
			b.curs.move(0, 1)
			b.curs.move(-curr_line_length, 0)
		} else if b.curs.x < b.contents[b.curs.y].Len() {
			// we have characters to the right, let's move along
			b.curs.move(1, 0)
		}
	case sdl.SCANCODE_LEFT:
		if b.curs.x == 0 && b.curs.y > 0 {
			b.curs.move(b.contents[b.curs.y-1].Len(), -1)

		} else if b.curs.x > 0 {
			b.curs.move(-1, 0)
		}
	case sdl.SCANCODE_TAB:
		if TABS_ARE_SPACES {
			// make an empty rune array of TAB_SIZE, cast to string
			// and insert it.
			b.contents[b.curs.y] = b.contents[b.curs.y].Insert(b.curs.x, "    ")
			b.curs.move(int(TAB_SIZE), 0)
		} else {
			b.contents[b.curs.y] = b.contents[b.curs.y].Insert(b.curs.x, string('\t'))
			// the actual position is + 1, but we make it
			// move by TAB_SIZE characters on the view.
			b.curs.move_render(1, 0, int(TAB_SIZE), 0)
		}
	}
}

func renderString(font *ttf.Font, val string, col sdl.Color, smooth bool) *sdl.Surface {
	if smooth {
		text, err := font.RenderUTF8_Blended(val, col)
		if err != nil {
			panic(err)
		}
		return text
	} else {
		text, err := font.RenderUTF8_Solid(val, col)
		if err != nil {
			panic(err)
		}
		return text
	}
	return nil
}

func (b *Buffer) Translate(x, y int32) {
	b.x += x
	b.y += y
}

func (b *Buffer) Update() {
	prev_x := b.curs.x
	prev_y := b.curs.y

	if b.input_handler == nil {
		panic("help")
	}

	if b.input_handler.Event != nil {
		switch t := b.input_handler.Event.(type) {
		case *sdl.TextInputEvent:
			b.processTextInput(t)
		case *sdl.KeyDownEvent:
			b.processActionKey(t)
		}
	}

	if b.curs.x != prev_x || b.curs.y != prev_y {
		should_draw = true
		should_flash = false
		reset_timer = sdl.GetTicks()
	}

	if !should_flash && sdl.GetTicks()-reset_timer > reset_delay_ms {
		should_flash = true
	}

	if sdl.GetTicks()-timer > cursor_flash_ms && should_flash {
		timer = sdl.GetTicks()
		should_draw = !should_draw
	}
}

var last_w, last_h int32
var TEXTURE_CACHE map[rune]*sdl.Texture = map[rune]*sdl.Texture{}

func (b *Buffer) Render(ctx *sdl.Renderer) {

	// render the ol' cursor
	if should_draw {
		gfx.SetDrawColorHex(ctx, 0x657B83)
		ctx.FillRect(&sdl.Rect{
			b.x + (int32(b.curs.rx)+1)*last_w,
			b.y + int32(b.curs.ry)*last_h,
			last_w,
			last_h,
		})
	}

	var y_col int32
	for _, rope := range b.contents {
		// this is because if we had the following
		// text input:
		//
		// Foo
		// _			<-- underscore is a space!
		// Blah
		// and we delete that underscore... it causes
		// a panic because there are no characters in
		// the empty string!
		if rope.Len() == 0 {
			// even though the string is empty
			// we still need to offset it by a line
			y_col += 1
			continue
		}

		var x_col int32
		for _, char := range rope.String() {
			switch char {
			case '\n':
				x_col = 0
				y_col += 1
				continue
			case '\t':
				x_col += TAB_SIZE
				continue
			}

			x_col += 1

			text := renderString(b.font, string(char), gfx.HexColor(0x7a7a7a), true)
			defer text.Free()

			last_w = text.W
			last_h = text.H

			texture, ok := TEXTURE_CACHE[char]
			if !ok {
				// can't find it in the cache so we
				// load and then cache it.
				texture, _ = ctx.CreateTextureFromSurface(text)
				TEXTURE_CACHE[char] = texture
			}

			// FIXME still kinda slow
			// we can also cull so that
			// we don't render things that aren't
			// visible outside of the component
			ctx.Copy(texture, nil, &sdl.Rect{
				b.x + (x_col * text.W),
				b.y + (y_col * text.H),
				text.W,
				text.H,
			})
		}

		y_col += 1
	}
}
<|endoftext|>


<commit_before>//
package file

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/nanobox-io/nanobox/config"
)

// Tar takes a source and variable writers and walks 'source' writing each file
// found to the tar writer; the purpose for accepting multiple writers is to allow
// for multiple outputs (for example a file, or md5 hash)
func Tar(src string, writers ...io.Writer) error {

	// ensure the src actually exists before trying to tar it
	if _, err := os.Stat(src); err != nil {
		return fmt.Errorf("Unable to tar files - %v", err.Error())
	}

	mw := io.MultiWriter(writers...)

	gzw := gzip.NewWriter(mw)
	defer gzw.Close()

	tw := tar.NewWriter(gzw)
	defer tw.Close()

	// walk path
	return filepath.Walk(src, func(file string, fi os.FileInfo, err error) error {

		// return on any error
		if err != nil {
			return err
		}

		// create a new dir/file header
		header, err := tar.FileInfoHeader(fi, fi.Name())
		if err != nil {
			return err
		}

		// update the name to correctly reflect the desired destination when untaring
		header.Name = strings.TrimPrefix(strings.Replace(file, src, "", -1), string(filepath.Separator))

		// write the header
		if err := tw.WriteHeader(header); err != nil {
			return err
		}

		// return on directories since there will be no content to tar
		if fi.Mode().IsDir() {
			return nil
		}

		// open files for taring
		f, err := os.Open(file)
		defer f.Close()
		if err != nil {
			return err
		}

		// copy file data into tar writer
		if _, err := io.Copy(tw, f); err != nil {
			return err
		}

		return nil
	})
}

// Untar takes a destination path and a reader; a tar reader loops over the tarfile
// creating the file structure at 'dst' along the way, and writing any files
func Untar(dst string, r io.Reader) error {

	gzr, err := gzip.NewReader(r)
	defer gzr.Close()
	if err != nil {
		return err
	}

	tr := tar.NewReader(gzr)

	for {
		header, err := tr.Next()

		switch {

		// if no more files are found return
		case err == io.EOF:
			return nil

		// return any other error
		case err != nil:
			return err

		// if the header is nil, just skip it (not sure how this happens)
		case header == nil:
			continue
		}

		// the target location where the dir/file should be created
		target := filepath.Join(dst, header.Name)

		// the following switch could also be done using fi.Mode(), not sure if there
		// a benefit of using one vs. the other.
		// fi := header.FileInfo()

		// check the file type
		switch header.Typeflag {

		// if its a dir and it doesn't exist create it
		case tar.TypeDir:
			if _, err := os.Stat(target); err != nil {
				if err := os.MkdirAll(target, 0755); err != nil {
					return err
				}
			}

		// if it's a file create it
		case tar.TypeReg:
			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
			if err != nil {
				return err
			}
			defer f.Close()

			// copy over contents
			if _, err := io.Copy(f, tr); err != nil {
				return err
			}
		}
	}
}

// Download downloads a file
func Download(path string, w io.Writer) error {
	res, err := http.Get(path)
	defer res.Body.Close()
	if err != nil {
		return err
	}

	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		config.Fatal("[util/file/file] ioutil.ReadAll() failed", err.Error())
	}

	w.Write(b)

	return nil
}

// Progress downloads a file with a fancy progress bar
func Progress(path string, w io.Writer) error {

	//
	download, err := http.Get(path)
	defer download.Body.Close()
	if err != nil {
		return err
	}

	var percent float64
	var down int

	// format the response content length to be more 'friendly'
	total := float64(download.ContentLength) / math.Pow(1024, 2)

	// create a 'buffer' to read into
	p := make([]byte, 2048)

	//
	for {

		// read the response body (streaming)
		n, err := download.Body.Read(p)

		// write to our buffer
		w.Write(p[:n])

		// update the total bytes read
		down += n

		switch {
		// return any other error
		default:
			// update the percent downloaded
			percent = (float64(down) / float64(download.ContentLength)) * 100

			// show download progress: 0.0/0.0MB [*** progress *** 0.0%]
			fmt.Printf("\r   %.2f/%.2fMB [%-41s %.2f%%]", float64(down)/math.Pow(1024, 2), total, strings.Repeat("*", int(percent/2.5)), percent)

		// if no more files are found return
		case download.ContentLength < 1:
			fmt.Printf("\r   %.2fMB", float64(down)/math.Pow(1024, 2))
		}

		// detect EOF and break the 'stream'
		if err != nil {
			if err == io.EOF {
				fmt.Println("")
				break
			} else {
				return err
			}
		}
	}

	return nil
}
<commit_msg>Update file.go<commit_after>//
package file

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/nanobox-io/nanobox/config"
)

// Tar takes a source and variable writers and walks 'source' writing each file
// found to the tar writer; the purpose for accepting multiple writers is to allow
// for multiple outputs (for example a file, or md5 hash)
func Tar(src string, writers ...io.Writer) error {

	// ensure the src actually exists before trying to tar it
	if _, err := os.Stat(src); err != nil {
		return fmt.Errorf("Unable to tar files - %v", err.Error())
	}

	mw := io.MultiWriter(writers...)

	gzw := gzip.NewWriter(mw)
	defer gzw.Close()

	tw := tar.NewWriter(gzw)
	defer tw.Close()

	// walk path
	return filepath.Walk(src, func(file string, fi os.FileInfo, err error) error {

		// return on any error
		if err != nil {
			return err
		}

		// create a new dir/file header
		header, err := tar.FileInfoHeader(fi, fi.Name())
		if err != nil {
			return err
		}

		// update the name to correctly reflect the desired destination when untaring
		header.Name = strings.TrimPrefix(strings.Replace(file, src, "", -1), string(filepath.Separator))

		// write the header
		if err := tw.WriteHeader(header); err != nil {
			return err
		}

		// return on directories since there will be no content to tar
		if fi.Mode().IsDir() {
			return nil
		}

		// open files for taring
		f, err := os.Open(file)
		defer f.Close()
		if err != nil {
			return err
		}

		// copy file data into tar writer
		if _, err := io.Copy(tw, f); err != nil {
			return err
		}

		return nil
	})
}

// Untar takes a destination path and a reader; a tar reader loops over the tarfile
// creating the file structure at 'dst' along the way, and writing any files
func Untar(dst string, r io.Reader) error {

	gzr, err := gzip.NewReader(r)
	defer gzr.Close()
	if err != nil {
		return err
	}

	tr := tar.NewReader(gzr)

	for {
		header, err := tr.Next()

		switch {

		// if no more files are found return
		case err == io.EOF:
			return nil

		// return any other error
		case err != nil:
			return err

		// if the header is nil, just skip it (not sure how this happens)
		case header == nil:
			continue
		}

		// the target location where the dir/file should be created
		target := filepath.Join(dst, header.Name)

		// the following switch could also be done using fi.Mode(), not sure if there
		// a benefit of using one vs. the other.
		// fi := header.FileInfo()

		// check the file type
		switch header.Typeflag {

		// if its a dir and it doesn't exist create it
		case tar.TypeDir:
			if _, err := os.Stat(target); err != nil {
				if err := os.MkdirAll(target, 0755); err != nil {
					return err
				}
			}

		// if it's a file create it
		case tar.TypeReg:
			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
			if err != nil {
				return err
			}
			defer f.Close()

			// copy over contents
			if _, err := io.Copy(f, tr); err != nil {
				return err
			}
		}
	}
}

// Download downloads a file
func Download(path string, w io.Writer) error {
	res, err := http.Get(path)
	defer res.Body.Close()
	if err != nil {
		return err
	}

	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		config.Fatal("[util/file/file] ioutil.ReadAll() failed", err.Error())
	}

	w.Write(b)

	return nil
}

// Progress downloads a file with a fancy progress bar
func Progress(path string, w io.Writer) error {

	//
	download, err := http.Get(path)
	defer download.Body.Close()
	if err != nil {
		return err
	}

	var percent float64
	var down int

	// format the response content length to be more 'friendly'
	total := float64(download.ContentLength) / math.Pow(1024, 2)

	// create a 'buffer' to read into
	p := make([]byte, 2048)

	//
	for {

		// read the response body (streaming)
		n, err := download.Body.Read(p)

		// write to our buffer
		w.Write(p[:n])

		// update the total bytes read
		down += n

		switch {
		default:
			// update the percent downloaded
			percent = (float64(down) / float64(download.ContentLength)) * 100

			// show download progress: 0.0/0.0MB [*** progress *** 0.0%]
			fmt.Printf("\r   %.2f/%.2fMB [%-41s %.2f%%]", float64(down)/math.Pow(1024, 2), total, strings.Repeat("*", int(percent/2.5)), percent)

		// if no more files are found return
		case download.ContentLength < 1:
			fmt.Printf("\r   %.2fMB", float64(down)/math.Pow(1024, 2))
		}

		// detect EOF and break the 'stream'
		if err != nil {
			if err == io.EOF {
				fmt.Println("")
				break
			} else {
				return err
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package pipe

import (
	"io"

	"github.com/yosisa/fluxion/event"
)

type Pipe interface {
	Read() (*event.Event, error)
	Write(*event.Event) error
}

type InProcess struct {
	c chan *event.Event
}

func NewInProcess() *InProcess {
	return &InProcess{c: make(chan *event.Event, 100)}
}

func (p *InProcess) Read() (*event.Event, error) {
	ev := <-p.c
	return ev, nil
}

func (p *InProcess) Write(ev *event.Event) error {
	p.c <- ev
	return nil
}

type InterProcess struct {
	enc event.Encoder
	dec event.Decoder
}

func NewInterProcess(r io.Reader, w io.Writer) *InterProcess {
	p := &InterProcess{}
	if r != nil {
		p.dec = event.NewDecoder(r)
	}
	if w != nil {
		p.enc = event.NewEncoder(w)
	}
	return p
}

func (p *InterProcess) Read() (*event.Event, error) {
	var ev event.Event
	if err := p.dec.Decode(&ev); err != nil {
		return nil, err
	}
	return &ev, nil
}

func (p *InterProcess) Write(ev *event.Event) error {
	return p.enc.Encode(ev)
}
<commit_msg>Fix race condition<commit_after>package pipe

import (
	"io"
	"sync"

	"github.com/yosisa/fluxion/event"
)

type Pipe interface {
	Read() (*event.Event, error)
	Write(*event.Event) error
}

type InProcess struct {
	c chan *event.Event
}

func NewInProcess() *InProcess {
	return &InProcess{c: make(chan *event.Event, 100)}
}

func (p *InProcess) Read() (*event.Event, error) {
	ev := <-p.c
	return ev, nil
}

func (p *InProcess) Write(ev *event.Event) error {
	p.c <- ev
	return nil
}

type InterProcess struct {
	enc event.Encoder
	dec event.Decoder
	rm  sync.Mutex
	wm  sync.Mutex
}

func NewInterProcess(r io.Reader, w io.Writer) *InterProcess {
	p := &InterProcess{}
	if r != nil {
		p.dec = event.NewDecoder(r)
	}
	if w != nil {
		p.enc = event.NewEncoder(w)
	}
	return p
}

func (p *InterProcess) Read() (*event.Event, error) {
	p.rm.Lock()
	defer p.rm.Unlock()

	var ev event.Event
	if err := p.dec.Decode(&ev); err != nil {
		return nil, err
	}
	return &ev, nil
}

func (p *InterProcess) Write(ev *event.Event) error {
	p.wm.Lock()
	defer p.wm.Unlock()
	return p.enc.Encode(ev)
}
<|endoftext|>


<commit_before>package radix

import (
	"strings"
	"sync"
	"time"
)

var blockingCmds = map[string]bool{
	"WAIT": true,

	// taken from https://github.com/joomcode/redispipe#limitations
	"BLPOP":      true,
	"BRPOP":      true,
	"BRPOPLPUSH": true,

	"BZPOPMIN": true,
	"BZPOPMAX": true,

	"XREAD":      true,
	"XREADGROUP": true,

	"SAVE":  true,
}

type pipeliner struct {
	c Client

	limit  int
	window time.Duration

	// reqsBufCh contains buffers for collecting commands and acts as a semaphore
	// to limit the number of concurrent flushes.
	reqsBufCh chan []CmdAction

	reqCh chan *pipelinerCmd
	reqWG sync.WaitGroup

	l      sync.RWMutex
	closed bool
}

var _ Client = (*pipeliner)(nil)

func newPipeliner(c Client, concurrency, limit int, window time.Duration) *pipeliner {
	if concurrency < 1 {
		concurrency = 1
	}

	p := &pipeliner{
		c: c,

		limit:  limit,
		window: window,

		reqsBufCh: make(chan []CmdAction, concurrency),

		reqCh: make(chan *pipelinerCmd, 32), // https://xkcd.com/221/
	}

	p.reqWG.Add(1)
	go func() {
		defer p.reqWG.Done()
		p.reqLoop()
	}()

	for i := 0; i < cap(p.reqsBufCh); i++ {
		if p.limit > 0 {
			p.reqsBufCh <- make([]CmdAction, 0, limit)
		} else {
			p.reqsBufCh <- nil
		}
	}

	return p
}

// CanDo checks if the given Action can be executed / passed to p.Do.
//
// If CanDo returns false, the Action must not be given to Do.
func (p *pipeliner) CanDo(a Action) bool {
	// there is currently no way to get the command for CmdAction implementations
	// from outside the radix package so we can not multiplex those commands. User
	// defined pipelines are not pipelined to let the user better control them.
	if cmdA, ok := a.(*cmdAction); ok {
		return !blockingCmds[strings.ToUpper(cmdA.cmd)]
	}
	return false
}

// Do executes the given Action as part of the pipeline.
//
// If a is not a CmdAction, Do panics.
func (p *pipeliner) Do(a Action) error {
	req := getPipelinerCmd(a.(CmdAction)) // get this outside the lock to avoid

	p.l.RLock()
	if p.closed {
		p.l.RUnlock()
		return errClientClosed
	}
	p.reqCh <- req
	p.l.RUnlock()

	err := <-req.resCh
	poolPipelinerCmd(req)
	return err
}

// Close closes the pipeliner and makes sure that all background goroutines
// are stopped before returning.
//
// Close does *not* close the underyling Client.
func (p *pipeliner) Close() error {
	p.l.Lock()
	defer p.l.Unlock()

	if p.closed {
		return nil
	}

	close(p.reqCh)
	p.reqWG.Wait()

	for i := 0; i < cap(p.reqsBufCh); i++ {
		<-p.reqsBufCh
	}

	p.c = nil
	return nil
}

func (p *pipeliner) reqLoop() {
	t := getTimer(time.Hour)
	defer putTimer(t)

	t.Stop()

	reqs := <-p.reqsBufCh
	defer func() {
		p.reqsBufCh <- reqs
	}()

	for {
		select {
		case req, ok := <-p.reqCh:
			if !ok {
				reqs = p.flush(reqs)
				return
			}

			reqs = append(reqs, req)

			if p.limit > 0 && len(reqs) == p.limit {
				// if we reached the pipeline limit, execute now to avoid unnecessary waiting
				t.Stop()

				reqs = p.flush(reqs)
			} else if len(reqs) == 1 {
				t.Reset(p.window)
			}
		case <-t.C:
			reqs = p.flush(reqs)
		}
	}
}

func (p *pipeliner) flush(reqs []CmdAction) []CmdAction {
	if len(reqs) == 0 {
		return reqs
	}

	go func() {
		defer func() {
			p.reqsBufCh <- reqs[:0]
		}()

		pipe := pipelinerPipeline{
			pipeline: pipeline(reqs),
		}

		if err := p.c.Do(pipe); err != nil {
			for _, req := range reqs {
				req.(*pipelinerCmd).resCh <- err
			}
		}
	}()

	return <-p.reqsBufCh
}

type pipelinerCmd struct {
	CmdAction
	resCh chan error
}

var pipelinerCmdPool sync.Pool

func getPipelinerCmd(cmd CmdAction) *pipelinerCmd {
	req, _ := pipelinerCmdPool.Get().(*pipelinerCmd)
	if req != nil {
		req.CmdAction = cmd
		return req
	}
	return &pipelinerCmd{
		CmdAction: cmd,
		// using a buffer of 1 is faster than no buffer in most cases
		resCh: make(chan error, 1),
	}
}

func poolPipelinerCmd(req *pipelinerCmd) {
	req.CmdAction = nil
	pipelinerCmdPool.Put(req)
}

type pipelinerPipeline struct {
	pipeline
}

func (p pipelinerPipeline) Run(c Conn) error {
	if err := c.Encode(p); err != nil {
		return err
	}
	for _, req := range p.pipeline {
		req.(*pipelinerCmd).resCh <- c.Decode(req)
	}
	return nil
}
<commit_msg>Set the closed flag on the pipeliner<commit_after>package radix

import (
	"strings"
	"sync"
	"time"
)

var blockingCmds = map[string]bool{
	"WAIT": true,

	// taken from https://github.com/joomcode/redispipe#limitations
	"BLPOP":      true,
	"BRPOP":      true,
	"BRPOPLPUSH": true,

	"BZPOPMIN": true,
	"BZPOPMAX": true,

	"XREAD":      true,
	"XREADGROUP": true,

	"SAVE":  true,
}

type pipeliner struct {
	c Client

	limit  int
	window time.Duration

	// reqsBufCh contains buffers for collecting commands and acts as a semaphore
	// to limit the number of concurrent flushes.
	reqsBufCh chan []CmdAction

	reqCh chan *pipelinerCmd
	reqWG sync.WaitGroup

	l      sync.RWMutex
	closed bool
}

var _ Client = (*pipeliner)(nil)

func newPipeliner(c Client, concurrency, limit int, window time.Duration) *pipeliner {
	if concurrency < 1 {
		concurrency = 1
	}

	p := &pipeliner{
		c: c,

		limit:  limit,
		window: window,

		reqsBufCh: make(chan []CmdAction, concurrency),

		reqCh: make(chan *pipelinerCmd, 32), // https://xkcd.com/221/
	}

	p.reqWG.Add(1)
	go func() {
		defer p.reqWG.Done()
		p.reqLoop()
	}()

	for i := 0; i < cap(p.reqsBufCh); i++ {
		if p.limit > 0 {
			p.reqsBufCh <- make([]CmdAction, 0, limit)
		} else {
			p.reqsBufCh <- nil
		}
	}

	return p
}

// CanDo checks if the given Action can be executed / passed to p.Do.
//
// If CanDo returns false, the Action must not be given to Do.
func (p *pipeliner) CanDo(a Action) bool {
	// there is currently no way to get the command for CmdAction implementations
	// from outside the radix package so we can not multiplex those commands. User
	// defined pipelines are not pipelined to let the user better control them.
	if cmdA, ok := a.(*cmdAction); ok {
		return !blockingCmds[strings.ToUpper(cmdA.cmd)]
	}
	return false
}

// Do executes the given Action as part of the pipeline.
//
// If a is not a CmdAction, Do panics.
func (p *pipeliner) Do(a Action) error {
	req := getPipelinerCmd(a.(CmdAction)) // get this outside the lock to avoid

	p.l.RLock()
	if p.closed {
		p.l.RUnlock()
		return errClientClosed
	}
	p.reqCh <- req
	p.l.RUnlock()

	err := <-req.resCh
	poolPipelinerCmd(req)
	return err
}

// Close closes the pipeliner and makes sure that all background goroutines
// are stopped before returning.
//
// Close does *not* close the underyling Client.
func (p *pipeliner) Close() error {
	p.l.Lock()
	defer p.l.Unlock()

	if p.closed {
		return nil
	}

	close(p.reqCh)
	p.reqWG.Wait()

	for i := 0; i < cap(p.reqsBufCh); i++ {
		<-p.reqsBufCh
	}

	p.c, p.closed = nil, true
	return nil
}

func (p *pipeliner) reqLoop() {
	t := getTimer(time.Hour)
	defer putTimer(t)

	t.Stop()

	reqs := <-p.reqsBufCh
	defer func() {
		p.reqsBufCh <- reqs
	}()

	for {
		select {
		case req, ok := <-p.reqCh:
			if !ok {
				reqs = p.flush(reqs)
				return
			}

			reqs = append(reqs, req)

			if p.limit > 0 && len(reqs) == p.limit {
				// if we reached the pipeline limit, execute now to avoid unnecessary waiting
				t.Stop()

				reqs = p.flush(reqs)
			} else if len(reqs) == 1 {
				t.Reset(p.window)
			}
		case <-t.C:
			reqs = p.flush(reqs)
		}
	}
}

func (p *pipeliner) flush(reqs []CmdAction) []CmdAction {
	if len(reqs) == 0 {
		return reqs
	}

	go func() {
		defer func() {
			p.reqsBufCh <- reqs[:0]
		}()

		pipe := pipelinerPipeline{
			pipeline: pipeline(reqs),
		}

		if err := p.c.Do(pipe); err != nil {
			for _, req := range reqs {
				req.(*pipelinerCmd).resCh <- err
			}
		}
	}()

	return <-p.reqsBufCh
}

type pipelinerCmd struct {
	CmdAction
	resCh chan error
}

var pipelinerCmdPool sync.Pool

func getPipelinerCmd(cmd CmdAction) *pipelinerCmd {
	req, _ := pipelinerCmdPool.Get().(*pipelinerCmd)
	if req != nil {
		req.CmdAction = cmd
		return req
	}
	return &pipelinerCmd{
		CmdAction: cmd,
		// using a buffer of 1 is faster than no buffer in most cases
		resCh: make(chan error, 1),
	}
}

func poolPipelinerCmd(req *pipelinerCmd) {
	req.CmdAction = nil
	pipelinerCmdPool.Put(req)
}

type pipelinerPipeline struct {
	pipeline
}

func (p pipelinerPipeline) Run(c Conn) error {
	if err := c.Encode(p); err != nil {
		return err
	}
	for _, req := range p.pipeline {
		req.(*pipelinerCmd).resCh <- c.Decode(req)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package util

import (
	"bytes"
	"testing"
)

var (
	sizeInt   int64 = 15351
	sizeBytes       = []byte{0, 0, 0x77, 0x77}
)

func TestParseSize(t *testing.T) {
	size, err := ParseSize(sizeBytes)
	if err != nil {
		t.Error(err)
	}
	if size != sizeInt {
		t.Errorf("Expected: %v, got: %v", sizeInt, size)
	}
}

func TestFormSize(t *testing.T) {
	size, err := FormSize(sizeInt)
	if err != nil {
		t.Error(err)
	}
	if !bytes.Equal(sizeBytes, size) {
		t.Errorf("Expected: %v, got: %v", sizeBytes, size)
	}
}
<commit_msg>Fix types in util tests<commit_after>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package util

import (
	"bytes"
	"testing"
)

var (
	sizeInt   int = 15351
	sizeBytes     = []byte{0, 0, 0x77, 0x77}
)

func TestParseSize(t *testing.T) {
	size, err := ParseSize(sizeBytes)
	if err != nil {
		t.Error(err)
	}
	if size != int64(sizeInt) {
		t.Errorf("Expected: %v, got: %v", sizeInt, size)
	}
}

func TestFormSize(t *testing.T) {
	size, err := FormSize(sizeInt)
	if err != nil {
		t.Error(err)
	}
	if !bytes.Equal(sizeBytes, size) {
		t.Errorf("Expected: %v, got: %v", sizeBytes, size)
	}
}
<|endoftext|>


<commit_before>package util

import (
	"bytes"
	"math/rand"
	"testing"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
)

func TestKey(t *testing.T) {

	h1, err := mh.Sum([]byte("beep boop"), mh.SHA2_256, -1)
	if err != nil {
		t.Error(err)
	}

	k1 := Key(h1)
	h2 := mh.Multihash(k1)
	k2 := Key(h2)

	if !bytes.Equal(h1, h2) {
		t.Error("Multihashes not equal.")
	}

	if k1 != k2 {
		t.Error("Keys not equal.")
	}
}

func TestByteChanReader(t *testing.T) {

	var data bytes.Buffer
	var data2 bytes.Buffer
	dch := make(chan []byte, 8)
	randr := NewTimeSeededRand()

	go func() {
		defer close(dch)
		for i := 0; i < rand.Intn(100)+100; i++ {
			chunk := make([]byte, rand.Intn(100000)+10)
			randr.Read(chunk)
			data.Write(chunk)
			// fmt.Printf("chunk: %6.d %v\n", len(chunk), chunk[:10])
			dch <- chunk
		}
	}()

	read := NewByteChanReader(dch)

	// read in random, weird sizes to exercise saving buffer.
	for {
		buf := make([]byte, rand.Intn(10)*10)
		n, err := read.Read(buf)
		data2.Write(buf[:n])
		// fmt.Printf("read: %6.d\n", n)
		if err != nil {
			break
		}
	}

	// fmt.Printf("lens: %d == %d\n", len(out), len(data.Bytes()))
	if !bytes.Equal(data2.Bytes(), data.Bytes()) {
		t.Fatal("Reader failed to stream correct bytes")
	}
}

func TestXOR(t *testing.T) {
	cases := [][3][]byte{
		[3][]byte{
			[]byte{0xFF, 0xFF, 0xFF},
			[]byte{0xFF, 0xFF, 0xFF},
			[]byte{0x00, 0x00, 0x00},
		},
		[3][]byte{
			[]byte{0x00, 0xFF, 0x00},
			[]byte{0xFF, 0xFF, 0xFF},
			[]byte{0xFF, 0x00, 0xFF},
		},
		[3][]byte{
			[]byte{0x55, 0x55, 0x55},
			[]byte{0x55, 0xFF, 0xAA},
			[]byte{0x00, 0xAA, 0xFF},
		},
	}

	for _, c := range cases {
		r := XOR(c[0], c[1])
		if !bytes.Equal(r, c[2]) {
			t.Error("XOR failed")
		}
	}
}
<commit_msg>rm comments<commit_after>package util

import (
	"bytes"
	"math/rand"
	"testing"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
)

func TestKey(t *testing.T) {

	h1, err := mh.Sum([]byte("beep boop"), mh.SHA2_256, -1)
	if err != nil {
		t.Error(err)
	}

	k1 := Key(h1)
	h2 := mh.Multihash(k1)
	k2 := Key(h2)

	if !bytes.Equal(h1, h2) {
		t.Error("Multihashes not equal.")
	}

	if k1 != k2 {
		t.Error("Keys not equal.")
	}
}

func TestByteChanReader(t *testing.T) {

	var data bytes.Buffer
	var data2 bytes.Buffer
	dch := make(chan []byte, 8)
	randr := NewTimeSeededRand()

	go func() {
		defer close(dch)
		for i := 0; i < rand.Intn(100)+100; i++ {
			chunk := make([]byte, rand.Intn(100000)+10)
			randr.Read(chunk)
			data.Write(chunk)
			dch <- chunk
		}
	}()

	read := NewByteChanReader(dch)

	// read in random, weird sizes to exercise saving buffer.
	for {
		buf := make([]byte, rand.Intn(10)*10)
		n, err := read.Read(buf)
		data2.Write(buf[:n])
		if err != nil {
			break
		}
	}

	if !bytes.Equal(data2.Bytes(), data.Bytes()) {
		t.Fatal("Reader failed to stream correct bytes")
	}
}

func TestXOR(t *testing.T) {
	cases := [][3][]byte{
		[3][]byte{
			[]byte{0xFF, 0xFF, 0xFF},
			[]byte{0xFF, 0xFF, 0xFF},
			[]byte{0x00, 0x00, 0x00},
		},
		[3][]byte{
			[]byte{0x00, 0xFF, 0x00},
			[]byte{0xFF, 0xFF, 0xFF},
			[]byte{0xFF, 0x00, 0xFF},
		},
		[3][]byte{
			[]byte{0x55, 0x55, 0x55},
			[]byte{0x55, 0xFF, 0xAA},
			[]byte{0x00, 0xAA, 0xFF},
		},
	}

	for _, c := range cases {
		r := XOR(c[0], c[1])
		if !bytes.Equal(r, c[2]) {
			t.Error("XOR failed")
		}
	}
}
<|endoftext|>


<commit_before>package junta

import (
	"os"
	"net"

	"junta/util"
	"junta/proto"
)

type conn struct {
	net.Conn
}

type server struct {
	net.Listener
}

func Serve(l net.Listener) os.Error {
	return (&server{l}).Serve()
}

func (s *server) Serve() os.Error {
	for {
		rw, e := s.Accept()
		if e != nil {
			return e
		}
		c := &conn{rw}
		go c.serve()
	}

	panic("not reached")
}

func (c *conn) serve() {
	pc := proto.NewConn(c)
	logger := util.NewLogger("%v", c.RemoteAddr())
	logger.Log("accepted connection")
	for {
		rid, parts, err := pc.ReadRequest()
		if err != nil {
			if err == os.EOF {
				logger.Log("connection closed by peer")
			} else {
				logger.Log(err)
			}
			return
		}

		rlogger := util.NewLogger("%v - req [%d]", c.RemoteAddr(), rid)
		rlogger.Logf("received <%v>", parts)

		if len(parts) == 0 {
			rlogger.Log("len(parts) == 0")
			rlogger.Log("before error")
			pc.SendError(rid, proto.InvalidCommand)
			rlogger.Log("after error")
			continue
		}

		switch parts[0] {
		default:
			rlogger.Logf("unknown command <%s>", parts[0])
			pc.SendError(rid, proto.InvalidCommand)
		case "set":
			//go set(c, rid, parts[1:])
			rlogger.Logf("set %q=%q", parts[1], parts[2])
			pc.SendResponse(rid, "OK")
		}
		rlogger.Log("bottom")
	}
}

func ListenAndServe(addr string) os.Error {
	logger := util.NewLogger("server %s", addr)

	logger.Logf("binding to %s", addr)
	l, err := net.Listen("tcp", addr)
	if err != nil {
		logger.Log(err)
		return err
	}
	defer l.Close()
	logger.Logf("listening on %s", addr)

	err = Serve(l)
	if err != nil {
		logger.Logf("%s: %s", l, err)
	}
	return err
}
<commit_msg>remove crufty logging<commit_after>package junta

import (
	"os"
	"net"

	"junta/util"
	"junta/proto"
)

type conn struct {
	net.Conn
}

type server struct {
	net.Listener
}

func Serve(l net.Listener) os.Error {
	return (&server{l}).Serve()
}

func (s *server) Serve() os.Error {
	for {
		rw, e := s.Accept()
		if e != nil {
			return e
		}
		c := &conn{rw}
		go c.serve()
	}

	panic("not reached")
}

func (c *conn) serve() {
	pc := proto.NewConn(c)
	logger := util.NewLogger("%v", c.RemoteAddr())
	logger.Log("accepted connection")
	for {
		rid, parts, err := pc.ReadRequest()
		if err != nil {
			if err == os.EOF {
				logger.Log("connection closed by peer")
			} else {
				logger.Log(err)
			}
			return
		}

		rlogger := util.NewLogger("%v - req [%d]", c.RemoteAddr(), rid)
		rlogger.Logf("received <%v>", parts)

		if len(parts) == 0 {
			rlogger.Log("zero parts supplied")
			pc.SendError(rid, proto.InvalidCommand)
			continue
		}

		switch parts[0] {
		default:
			rlogger.Logf("unknown command <%s>", parts[0])
			pc.SendError(rid, proto.InvalidCommand)
		case "set":
			//go set(c, rid, parts[1:])
			rlogger.Logf("set %q=%q", parts[1], parts[2])
			pc.SendResponse(rid, "OK")
		}
	}
}

func ListenAndServe(addr string) os.Error {
	logger := util.NewLogger("server %s", addr)

	logger.Logf("binding to %s", addr)
	l, err := net.Listen("tcp", addr)
	if err != nil {
		logger.Log(err)
		return err
	}
	defer l.Close()
	logger.Logf("listening on %s", addr)

	err = Serve(l)
	if err != nil {
		logger.Logf("%s: %s", l, err)
	}
	return err
}
<|endoftext|>


<commit_before>package main

type HEHandler struct{}

func (handler *HEHandler) DomainLoop(domain *Domain) {
}
<commit_msg>implement HE dns handler<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime/debug"
	"time"

	"github.com/parnurzeal/gorequest"
)

var (
	HEUrl = "https://dyn.dns.he.net/nic/update?hostname=%s.%s&password=%s&myip=%s"
)

type HEHandler struct{}

func (handler *HEHandler) DomainLoop(domain *Domain) {
	defer func() {
		if err := recover(); err != nil {
			panicCount++
			log.Printf("Recovered in %v: %v\n", err, debug.Stack())
			fmt.Println(identifyPanic())
			log.Print(identifyPanic())
			if panicCount < PANIC_MAX {
				log.Println("Got panic in goroutine, will start a new one... :", panicCount)
				go handler.DomainLoop(domain)
			} else {
				os.Exit(1)
			}
		}
	}()

	for {
		currentIP, err := getCurrentIP(configuration.IPUrl)

		if err != nil {
			log.Println("get_currentIP:", err)
			continue
		}
		log.Println("currentIp is:", currentIP)

		for _, subDomain := range domain.SubDomains {
			log.Printf("%s.%s Start to update record IP...\n", subDomain, domain.DomainName)
			handler.UpdateIP(domain.DomainName, subDomain, currentIP)
		}

		//Interval is 5 minutes
		time.Sleep(time.Minute * INTERVAL)
	}
}

func (handler *HEHandler) UpdateIP(domain, subDomain, currentIP string) {
	request := gorequest.New()
	resp, body, errs := request.Get(fmt.Sprintf(HEUrl, subDomain, domain, configuration.Password, currentIP)).End()

	if len(errs) > 0 {
		log.Println("Request error...")

		for _, err := range errs {
			log.Println("Err:", err.Error())
		}
	} else {
		if resp.StatusCode == http.StatusOK {
			log.Println("Update IP success:", body)
		} else {
			log.Println("Update IP failed:", body)
		}
	}
}
<|endoftext|>


<commit_before>package main
import (
        "fmt"
        "net/http"
        "os"
)
func handler(w http.ResponseWriter, r *http.Request) {
        h, _ := os.Hostname()
        fmt.Fprintf(w, "Hi there, I'm served from %s!", h)
}
func main() {
        http.HandleFunc("/", handler)
        http.ListenAndServe(":8484", nil)
}
<commit_msg>Change hellow world to use name argument if added<commit_after>package main
import (
        "fmt"
        "net/http"
        "os"
)

func handler(w http.ResponseWriter, r *http.Request) {
        h, _ := os.Hostname()
	if len(r.URL.Query().Get("name")) <= 0  {
       	   fmt.Fprintf(w, "Hi there, I'm served from %s!", h)
        } else {
       	   fmt.Fprintf(w, "Hi there %s, I'm served from %s!", r.URL.Query().Get("name"), h)
        }
}

func main() {
        http.HandleFunc("/", handler)
        http.ListenAndServe(":8484", nil)
}
<|endoftext|>


<commit_before>package page

import (
	"encoding/json"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"net/http"
	"net/url"
)

// Page represents HTTP response page
type Page struct {
	Trace Trace

	URL        *url.URL
	StatusCode int
	Header     http.Header
	Body       []byte

	document *goquery.Document
}

// IsEmpty returns true if page byte content is empty
func (p Page) IsEmpty() bool {
	return len(p.Body) == 0
}

// ContentType returns content type of page and "application/octet-stream" as default
func (p Page) ContentType() string {
	ct := p.Header.Get("content-type")
	if len(ct) == 0 {
		return "application/octet-stream"
	}

	return ct
}

// String returns short string representation of page
func (p Page) String() string {
	return fmt.Sprintf(
		"HTML [%d] %s %s with %d bytes fetched in [%s]",
		p.StatusCode,
		p.URL.String(),
		p.ContentType(),
		len(p.Body),
		p.Trace.String(),
	)
}

// AsString converts body to string
func (p Page) AsString() string {
	return string(p.Body)
}

// AsJSON unmarshals data bytes into struct using JSON
func (p Page) AsJSON(target interface{}) error {
	return json.Unmarshal(p.Body, target)
}
<commit_msg>page json serialization<commit_after>package page

import (
	"encoding/json"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"net/http"
	"net/url"
)

// Page represents HTTP response page
type Page struct {
	Trace Trace

	URL        *url.URL
	StatusCode int
	Header     http.Header
	Body       []byte

	document *goquery.Document
}

// IsEmpty returns true if page byte content is empty
func (p Page) IsEmpty() bool {
	return len(p.Body) == 0
}

// ContentType returns content type of page and "application/octet-stream" as default
func (p Page) ContentType() string {
	ct := p.Header.Get("content-type")
	if len(ct) == 0 {
		return "application/octet-stream"
	}

	return ct
}

// String returns short string representation of page
func (p Page) String() string {
	return fmt.Sprintf(
		"HTML [%d] %s %s with %d bytes fetched in [%s]",
		p.StatusCode,
		p.URL.String(),
		p.ContentType(),
		len(p.Body),
		p.Trace.String(),
	)
}

// AsString converts body to string
func (p Page) AsString() string {
	return string(p.Body)
}

// AsJSON unmarshals data bytes into struct using JSON
func (p Page) AsJSON(target interface{}) error {
	return json.Unmarshal(p.Body, target)
}

// MarshalJSON is implementation of JSON parsing helpers
func (p Page) MarshalJSON() ([]byte, error) {
	m := map[string]interface{}{}
	m["URL"] = p.URL.String()
	m["StatusCode"] = p.StatusCode
	m["Header"] = p.Header
	m["Body"] = string(p.Body)

	return json.Marshal(m)
}

// UnmarshalJSON is implementation of JSON parsing helpers
func (p *Page) UnmarshalJSON(data []byte) error {
	var m struct {
		URL        string              `json:"URL"`
		StatusCode int                 `json:"StatusCode"`
		Header     map[string][]string `json:"Header"`
		Body       string              `json:"body"`
	}
	err := json.Unmarshal(data, &m)
	if err != nil {
		return err
	}

	url, err := url.Parse(m.URL)
	if err != nil {
		return err
	}

	p.URL = url
	p.Header = http.Header(m.Header)
	p.StatusCode = m.StatusCode
	p.Body = []byte(m.Body)

	return nil
}
<|endoftext|>


<commit_before>package models

import (
	"errors"
	"fmt"
	"reflect"
	"strings"

	"github.com/astaxie/beego/orm"
)

type Vehiculo struct {
	Id            int          `orm:"column(id);pk;auto"`
	Placa         string       `orm:"column(placa)"`
	IdNfc         int16        `orm:"column(id_nfc)"`
	IdPropietario *Propietario `orm:"column(id_propietario);rel(fk)"`
}

func (t *Vehiculo) TableName() string {
	return "vehiculo"
}

func init() {
	orm.RegisterModel(new(Vehiculo))
}

// AddVehiculo insert a new Vehiculo into database and returns
// last inserted Id on success.
func AddVehiculo(m *Vehiculo) (id int64, err error) {
	o := orm.NewOrm()
	m.IdPropietario, _ = GetPropietarioById(m.IdPropietario.Id)
	id, err = o.Insert(m)
	return
}

// GetVehiculoById retrieves Vehiculo by Id. Returns error if
// Id doesn't exist
func GetVehiculoById(id int) (v *Vehiculo, err error) {
	o := orm.NewOrm()
	v = &Vehiculo{Id: id}
	if err = o.Read(v); err == nil {
		v.IdPropietario, _ = GetPropietarioById(v.IdPropietario.Id)
		return v, nil
	}
	return nil, err
}

// GetAllVehiculo retrieves all Vehiculo matches certain condition. Returns empty list if
// no records exist
func GetAllVehiculo(query map[string]string, fields []string, sortby []string, order []string,
	offset int64, limit int64) (ml []interface{}, err error) {
	o := orm.NewOrm()
	qs := o.QueryTable(new(Vehiculo))
	// query k=v
	for k, v := range query {
		// rewrite dot-notation to Object__Attribute
		k = strings.Replace(k, ".", "__", -1)
		qs = qs.Filter(k, v)
	}
	// order by:
	var sortFields []string
	if len(sortby) != 0 {
		if len(sortby) == len(order) {
			// 1) for each sort field, there is an associated order
			for i, v := range sortby {
				orderby := ""
				if order[i] == "desc" {
					orderby = "-" + v
				} else if order[i] == "asc" {
					orderby = v
				} else {
					return nil, errors.New("Error: Invalid order. Must be either [asc|desc]")
				}
				sortFields = append(sortFields, orderby)
			}
			qs = qs.OrderBy(sortFields...)
		} else if len(sortby) != len(order) && len(order) == 1 {
			// 2) there is exactly one order, all the sorted fields will be sorted by this order
			for _, v := range sortby {
				orderby := ""
				if order[0] == "desc" {
					orderby = "-" + v
				} else if order[0] == "asc" {
					orderby = v
				} else {
					return nil, errors.New("Error: Invalid order. Must be either [asc|desc]")
				}
				sortFields = append(sortFields, orderby)
			}
		} else if len(sortby) != len(order) && len(order) != 1 {
			return nil, errors.New("Error: 'sortby', 'order' sizes mismatch or 'order' size is not 1")
		}
	} else {
		if len(order) != 0 {
			return nil, errors.New("Error: unused 'order' fields")
		}
	}

	var l []Vehiculo
	qs = qs.OrderBy("id")
	if _, err := qs.Limit(limit, offset).All(&l, fields...); err == nil {
		if len(fields) == 0 {
			for _, v := range l {
				v.IdPropietario, _ = GetPropietarioById(v.IdPropietario.Id)
				ml = append(ml, v)
			}
		} else {
			// trim unused fields
			for _, v := range l {
				m := make(map[string]interface{})
				val := reflect.ValueOf(v)
				for _, fname := range fields {
					m[fname] = val.FieldByName(fname).Interface()
				}
				ml = append(ml, m)
			}
		}
		return ml, nil
	}
	return nil, err
}

// UpdateVehiculo updates Vehiculo by Id and returns error if
// the record to be updated doesn't exist
func UpdateVehiculoById(m *Vehiculo) (err error) {
	o := orm.NewOrm()
	v := Vehiculo{Id: m.Id}
	// ascertain id exists in the database
	if err = o.Read(&v); err == nil {
		var num int64
		if num, err = o.Update(m); err == nil {
			fmt.Println("Number of records updated in database:", num)
		}
	}
	return
}

// DeleteVehiculo deletes Vehiculo by Id and returns error if
// the record to be deleted doesn't exist
func DeleteVehiculo(id int) (err error) {
	o := orm.NewOrm()
	v := Vehiculo{Id: id}
	// ascertain id exists in the database
	if err = o.Read(&v); err == nil {
		var num int64
		if num, err = o.Delete(&Vehiculo{Id: id}); err == nil {
			fmt.Println("Number of records deleted in database:", num)
		}
	}
	return
}
<commit_msg>Se anexa el mecanismo de validacion de datos para vehiculo, se omite tipo Int y para llaves foraneas por misma restriccion<commit_after>package models

import (
	"errors"
	"fmt"
	"log"
	"reflect"
	"strings"

	"github.com/astaxie/beego/orm"
	"github.com/astaxie/beego/validation"
)

type Vehiculo struct {
	Id            int          `orm:"column(id);pk;auto"`
	Placa         string       `orm:"column(placa)"`
	IdNfc         int16        `orm:"column(id_nfc)"`
	IdPropietario *Propietario `orm:"column(id_propietario);rel(fk)"`
}

func (t *Vehiculo) TableName() string {
	return "vehiculo"
}

func init() {
	orm.RegisterModel(new(Vehiculo))
}

// AddVehiculo insert a new Vehiculo into database and returns
// last inserted Id on success.
func AddVehiculo(m *Vehiculo) (id int64, err error) {
	o := orm.NewOrm()
	valid := validation.Validation{}
	valid.Required(m.Placa, "Placa")
	valid.MaxSize(m.Placa, 6, "PlacaMax")
	valid.Required(m.IdNfc, "IdNfc")
	valid.AlphaNumeric(m.Placa, "PlacaAlphaNum")
	if valid.HasErrors() {
		for _, err := range valid.Errors {
			log.Println(err.Key, err.Message)
		}
	} else {
		log.Println("Insert New Register")
		m.IdPropietario, _ = GetPropietarioById(m.IdPropietario.Id)
		id, err = o.Insert(m)
	}
	return
}

// GetVehiculoById retrieves Vehiculo by Id. Returns error if
// Id doesn't exist
func GetVehiculoById(id int) (v *Vehiculo, err error) {
	o := orm.NewOrm()
	v = &Vehiculo{Id: id}
	if err = o.Read(v); err == nil {
		v.IdPropietario, _ = GetPropietarioById(v.IdPropietario.Id)
		return v, nil
	}
	return nil, err
}

// GetAllVehiculo retrieves all Vehiculo matches certain condition. Returns empty list if
// no records exist
func GetAllVehiculo(query map[string]string, fields []string, sortby []string, order []string,
	offset int64, limit int64) (ml []interface{}, err error) {
	o := orm.NewOrm()
	qs := o.QueryTable(new(Vehiculo))
	// query k=v
	for k, v := range query {
		// rewrite dot-notation to Object__Attribute
		k = strings.Replace(k, ".", "__", -1)
		qs = qs.Filter(k, v)
	}
	// order by:
	var sortFields []string
	if len(sortby) != 0 {
		if len(sortby) == len(order) {
			// 1) for each sort field, there is an associated order
			for i, v := range sortby {
				orderby := ""
				if order[i] == "desc" {
					orderby = "-" + v
				} else if order[i] == "asc" {
					orderby = v
				} else {
					return nil, errors.New("Error: Invalid order. Must be either [asc|desc]")
				}
				sortFields = append(sortFields, orderby)
			}
			qs = qs.OrderBy(sortFields...)
		} else if len(sortby) != len(order) && len(order) == 1 {
			// 2) there is exactly one order, all the sorted fields will be sorted by this order
			for _, v := range sortby {
				orderby := ""
				if order[0] == "desc" {
					orderby = "-" + v
				} else if order[0] == "asc" {
					orderby = v
				} else {
					return nil, errors.New("Error: Invalid order. Must be either [asc|desc]")
				}
				sortFields = append(sortFields, orderby)
			}
		} else if len(sortby) != len(order) && len(order) != 1 {
			return nil, errors.New("Error: 'sortby', 'order' sizes mismatch or 'order' size is not 1")
		}
	} else {
		if len(order) != 0 {
			return nil, errors.New("Error: unused 'order' fields")
		}
	}

	var l []Vehiculo
	qs = qs.OrderBy("id")
	if _, err := qs.Limit(limit, offset).All(&l, fields...); err == nil {
		if len(fields) == 0 {
			for _, v := range l {
				v.IdPropietario, _ = GetPropietarioById(v.IdPropietario.Id)
				ml = append(ml, v)
			}
		} else {
			// trim unused fields
			for _, v := range l {
				m := make(map[string]interface{})
				val := reflect.ValueOf(v)
				for _, fname := range fields {
					m[fname] = val.FieldByName(fname).Interface()
				}
				ml = append(ml, m)
			}
		}
		return ml, nil
	}
	return nil, err
}

// UpdateVehiculo updates Vehiculo by Id and returns error if
// the record to be updated doesn't exist
func UpdateVehiculoById(m *Vehiculo) (err error) {
	o := orm.NewOrm()
	v := Vehiculo{Id: m.Id}
	// ascertain id exists in the database
	if err = o.Read(&v); err == nil {
		var num int64
		if num, err = o.Update(m); err == nil {
			fmt.Println("Number of records updated in database:", num)
		}
	}
	return
}

// DeleteVehiculo deletes Vehiculo by Id and returns error if
// the record to be deleted doesn't exist
func DeleteVehiculo(id int) (err error) {
	o := orm.NewOrm()
	v := Vehiculo{Id: id}
	// ascertain id exists in the database
	if err = o.Read(&v); err == nil {
		var num int64
		if num, err = o.Delete(&Vehiculo{Id: id}); err == nil {
			fmt.Println("Number of records deleted in database:", num)
		}
	}
	return
}
<|endoftext|>


<commit_before>package ingestor

import (
	"encoding/json"
	"net/http"

	"github.com/google/go-github/github"
	"github.com/pivotal-golang/lager"
)

type handler struct {
	logger    lager.Logger
	secretKey []byte
	ingestor  Ingestor
}

func Handler(logger lager.Logger, ingestor Ingestor, secretKey string) *handler {
	return &handler{
		logger:    logger.Session("webhook-handler"),
		secretKey: []byte(secretKey),
		ingestor:  ingestor,
	}
}

func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	payload, err := github.ValidatePayload(r, h.secretKey)
	if err != nil {
		h.logger.Error("invalid-payload", err)
		w.WriteHeader(http.StatusForbidden)
		return
	}

	var event github.PushEvent
	if err := json.Unmarshal(payload, &event); err != nil {
		h.logger.Error("unmarshal-failed", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	err = h.handlePushEvent(h.logger, w, event, r.Header.Get("X-GitHub-Delivery"))
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (h *handler) handlePushEvent(
	logger lager.Logger,
	w http.ResponseWriter,
	event github.PushEvent,
	gitHubID string,
) error {
	logger = logger.Session("handling-push-event")

	scan, valid := Extract(event)
	if !valid {
		if event.Repo != nil && event.Repo.FullName != nil {
			logger = logger.WithData(lager.Data{
				"repo": *event.Repo.FullName,
			})
		}

		logger.Debug("event-missing-data")
		return nil
	}

	logger.Info("handling-webhook-payload", lager.Data{
		"before":  scan.From,
		"after":   scan.To,
		"repo":    scan.FullRepoName(),
		"private": scan.Private,
	})

	return h.ingestor.IngestPushScan(logger, scan, gitHubID)
}
<commit_msg>inline event handling in webhook<commit_after>package ingestor

import (
	"encoding/json"
	"net/http"

	"github.com/google/go-github/github"
	"github.com/pivotal-golang/lager"
)

type handler struct {
	logger    lager.Logger
	secretKey []byte
	ingestor  Ingestor
}

func Handler(logger lager.Logger, ingestor Ingestor, secretKey string) *handler {
	return &handler{
		logger:    logger.Session("webhook-handler"),
		secretKey: []byte(secretKey),
		ingestor:  ingestor,
	}
}

func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	payload, err := github.ValidatePayload(r, h.secretKey)
	if err != nil {
		h.logger.Error("invalid-payload", err)
		w.WriteHeader(http.StatusForbidden)
		return
	}

	var event github.PushEvent
	if err := json.Unmarshal(payload, &event); err != nil {
		h.logger.Error("unmarshal-failed", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	scan, valid := Extract(event)
	if !valid {
		h.logger.Debug("event-missing-data", lager.Data{
			"repo": *event.Repo.FullName,
		})
		w.WriteHeader(http.StatusOK)
		return
	}

	h.logger.Info("handling-webhook-payload", lager.Data{
		"before":  scan.From,
		"after":   scan.To,
		"owner":   scan.Owner,
		"repo":    scan.Repository,
		"private": scan.Private,
	})

	err = h.ingestor.IngestPushScan(h.logger, scan, r.Header.Get("X-GitHub-Delivery"))
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}
<|endoftext|>


<commit_before>package pair

import (
	zmq "github.com/pebbe/zmq3"
)

type Conn struct {
	OnMethod    chan zmqMessage
	OnClose     chan (bool)
	sendMessage chan zmqMessage
	pair        *zmq.Socket
	addr        string
}

type zmqMessage struct {
	Method string
	Params []byte
}

// NewConn creates a Conn(ection) which includes callback channels on which you
// must select.
func NewConn() *Conn {
	sock, _ := zmq.NewSocket(zmq.PAIR)
	return &Conn{make(chan zmqMessage), make(chan bool), make(chan zmqMessage), sock, ""}
}

// Connnect the underlying pair connection to the given address
func (z *Conn) Connect(addr string) {
	z.addr = addr
	z.pair.Connect(addr)
}

// Send a message
func (z *Conn) Send(method string, params []byte) {
	z.sendMessage <- zmqMessage{method, params}
}

// Run starts reading on the pair connection, and sending messages.
// Requires a background goroutine for reading.
func (z *Conn) Run() {
	sock := z.pair

	// Set a hwm of 1 - it's pointless to buffer requests for stats, and this
	// means that we get an error when we try to send if the client has gone
	sock.SetSndhwm(1)

	// Removing because it causes crash... (due to read goroutine)
	// defer sock.Close()

	// These are used so that we can stop the reading goroutine - hacky...
	sock.SetRcvtimeo(61e9) // 61s
	closed := false

	// Used to signal shutdown to the main goroutine
	sig_shutdown := make(chan bool)

	// TODO: This isn't correct because sock not threadsafe...
	go func() {
		defer func() {
			debug.Printf("[zmqclient] Closing socket")
			sock.Close()
		}()

		for {
			parts, err := sock.RecvMessageBytes(0)

			if closed {
				// Returns from this (zmq reading) goroutine
				return
			}

			if err != nil {
				debug.Printf("[zmqclient] Recv err: %v", err)
			} else {
				s := string(parts[0])

				// Non-prefixed ping & ping are deprecated. TODO: remove
				if s == "ping" || s == "pair:ping" {
					debug.Print("[zmqclient] Received ping, sending pong")
					z.Send("pair:pong", []byte(""))
				} else if s == "pong" || s == "pair:pong" {
					debug.Print("[zmqclient] Received pong")
					// Do nothing
				} else if s == "pair:shutdown" {
					debug.Print("[zmqclient] Received shutdown")
					sig_shutdown <- true
				} else {
					if len(parts) != 2 {
						info.Printf("[zmqclient] Unepected message %v with %v parts", s, len(parts))
					} else {
						debug.Printf("[zmqclient] Received message %s", s)
						z.OnMethod <- zmqMessage{s, parts[1]}
					}
				}
			}
		}
	}()

	defer func() {
		closed = true
		z.OnClose <- true
	}()

	for {
		select {
		case <-sig_shutdown:
			return
		case msg := <-z.sendMessage:
			// Note: I considered using sock.SendMessage but unfortunately that
			// doesn't support arbitrary flags, and it basically does this anway.

			// Set DONTWAIT so that Send doesn't block when HWM is reached

			// In case of an error and return, an OnClose will be sent by defer

			if _, err := sock.Send(msg.Method, zmq.DONTWAIT|zmq.SNDMORE); err != nil {
				info.Printf("[zmqclient] Closing %v after err: %v", z.addr, err)
				return
			}
			if _, err := sock.SendBytes(msg.Params, zmq.DONTWAIT); err != nil {
				info.Printf("[zmqclient] Closing %v after err: %v", z.addr, err)
				return
			}
		}
	}
}
<commit_msg>Bugfix - zmq HWM must be set before connect/bind<commit_after>package pair

import (
	zmq "github.com/pebbe/zmq3"
)

type Conn struct {
	OnMethod    chan zmqMessage
	OnClose     chan (bool)
	sendMessage chan zmqMessage
	pair        *zmq.Socket
	addr        string
}

type zmqMessage struct {
	Method string
	Params []byte
}

// NewConn creates a Conn(ection) which includes callback channels on which you
// must select.
func NewConn() *Conn {
	sock, _ := zmq.NewSocket(zmq.PAIR)

	// Set a HWM to force an error when sending, thereby detecting that the
	// client has gone away
	if err := sock.SetSndhwm(1); err != nil {
		info.Printf("[pair] Error setting HWM: %v", err)
	}

	return &Conn{make(chan zmqMessage), make(chan bool), make(chan zmqMessage), sock, ""}
}

// Connnect the underlying pair connection to the given address
func (z *Conn) Connect(addr string) {
	z.addr = addr
	z.pair.Connect(addr)
}

// Send a message
func (z *Conn) Send(method string, params []byte) {
	z.sendMessage <- zmqMessage{method, params}
}

// Run starts reading on the pair connection, and sending messages.
// Requires a background goroutine for reading.
func (z *Conn) Run() {
	sock := z.pair

	// Removing because it causes crash... (due to read goroutine)
	// defer sock.Close()

	// These are used so that we can stop the reading goroutine - hacky...
	sock.SetRcvtimeo(61e9) // 61s
	closed := false

	// Used to signal shutdown to the main goroutine
	sig_shutdown := make(chan bool)

	// TODO: This isn't correct because sock not threadsafe...
	go func() {
		defer func() {
			debug.Printf("[zmqclient] Closing socket")
			sock.Close()
		}()

		for {
			parts, err := sock.RecvMessageBytes(0)

			if closed {
				// Returns from this (zmq reading) goroutine
				return
			}

			if err != nil {
				debug.Printf("[zmqclient] Recv err: %v", err)
			} else {
				s := string(parts[0])

				// Non-prefixed ping & ping are deprecated. TODO: remove
				if s == "ping" || s == "pair:ping" {
					debug.Print("[zmqclient] Received ping, sending pong")
					z.Send("pair:pong", []byte(""))
				} else if s == "pong" || s == "pair:pong" {
					debug.Print("[zmqclient] Received pong")
					// Do nothing
				} else if s == "pair:shutdown" {
					debug.Print("[zmqclient] Received shutdown")
					sig_shutdown <- true
				} else {
					if len(parts) != 2 {
						info.Printf("[zmqclient] Unepected message %v with %v parts", s, len(parts))
					} else {
						debug.Printf("[zmqclient] Received message %s", s)
						z.OnMethod <- zmqMessage{s, parts[1]}
					}
				}
			}
		}
	}()

	defer func() {
		closed = true
		z.OnClose <- true
	}()

	for {
		select {
		case <-sig_shutdown:
			return
		case msg := <-z.sendMessage:
			// Note: I considered using sock.SendMessage but unfortunately that
			// doesn't support arbitrary flags, and it basically does this anway.

			// Set DONTWAIT so that Send doesn't block when HWM is reached

			// In case of an error and return, an OnClose will be sent by defer

			if _, err := sock.Send(msg.Method, zmq.DONTWAIT|zmq.SNDMORE); err != nil {
				info.Printf("[zmqclient] Closing %v after err: %v", z.addr, err)
				return
			}
			if _, err := sock.SendBytes(msg.Params, zmq.DONTWAIT); err != nil {
				info.Printf("[zmqclient] Closing %v after err: %v", z.addr, err)
				return
			}
		}
	}
}
<|endoftext|>


<commit_before>package memaccess

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

type process struct {
	pid          uint
	mapsFilepath string
	memFilepath  string
}

func newProcessMemoryReaderImpl(pid uint) (process, error, []error) {
	var result process
	result.pid = pid
	result.mapsFilepath = filepath.Join("/proc", fmt.Sprintf("%d", pid), "maps")
	result.memFilepath = filepath.Join("/proc", fmt.Sprintf("%d", pid), "mem")
	softerrors := make([]error, 0)

	// trying to open the maps file, only to see if we have enought privileges
	f, harderror := os.Open(result.mapsFilepath)
	if harderror != nil {
		return process{}, harderror, softerrors
	}
	f.Close()

	return result, nil, softerrors
}

func (p process) Close() (error, []error) {
	return nil, make([]error, 0)
}

func (p process) NextReadableMemoryRegion(address uintptr) (region MemoryRegion, harderror error, softerrors []error) {
	softerrors = make([]error, 0)

	mapsFile, harderror := os.Open(p.mapsFilepath)
	if harderror != nil {
		return
	}
	defer mapsFile.Close()

	region = MemoryRegion{}
	scanner := bufio.NewScanner(mapsFile)
	splitBySpacesRegexp := regexp.MustCompile("\\s+")

	for scanner.Scan() {
		line := scanner.Text()
		items := splitBySpacesRegexp.Split(line, -1)

		if len(items) != 6 {
			return region, fmt.Errorf("Unrecognised maps line: %s", line), softerrors
		}

		start, end, err := parseMemoryLimits(items[0])
		if err != nil {
			return region, err, softerrors
		}

		if end <= address {
			continue
		}

		// Skip vsyscall as it can't be read. It's a special page mapped by the kernel to accelerate some syscalls.
		if items[5] == "[vsyscall]" {
			continue
		}

		// Check if memory is unreadable
		if items[1][0] == '-' {

			// If we were already reading a region this will just finish it. We only report the softerror when we
			// were actually trying to read it.
			if region.Address != 0 {
				return region, nil, softerrors
			}

			softerrors = append(softerrors, fmt.Errorf("Unreadable memory %s", items[0]))
			continue
		}

		size := uint(end - start)

		// Begenning of a region
		if region.Address == 0 {
			region = MemoryRegion{Address: start, Size: size}
			continue
		}

		// Continuation of a region
		if region.Address+uintptr(region.Size) == start {
			region.Size += size
			continue
		}

		// This map is outside the current region, so we are ready
		return region, nil, softerrors
	}

	// No region left
	if err := scanner.Err(); err != nil {
		return region, err, softerrors
	}

	return NoRegionAvailable, nil, softerrors
}

func (p process) CopyMemory(address uintptr, buffer []byte) (error, []error) {
	mem, harderror := os.Open(p.memFilepath)
	softerrors := make([]error, 0)
	if harderror != nil {
		harderror := fmt.Errorf("Error while reading %d bytes starting at %x: %s", len(buffer), address, harderror)
		return harderror, softerrors
	}
	defer mem.Close()

	bytes_read, harderror := mem.ReadAt(buffer, int64(address))
	if harderror != nil {
		harderror := fmt.Errorf("Error while reading %d bytes starting at %x: %s", len(buffer), address, harderror)
		return harderror, softerrors
	}

	if bytes_read != len(buffer) {
		return fmt.Errorf("Coul not read the entire buffer"), softerrors
	}

	return nil, softerrors
}

//Parses the memory limits of a mapping as found in /proc/PID/maps
func parseMemoryLimits(limits string) (start uintptr, end uintptr, err error) {
	fields := strings.Split(limits, "-")
	start64, err := strconv.ParseUint(fields[0], 16, 64)
	if err != nil {
		return 0, 0, err
	}
	start = uintptr(start64)

	end64, err := strconv.ParseUint(fields[1], 16, 64)
	if err != nil {
		return 0, 0, err
	}
	end = uintptr(end64)

	return
}
<commit_msg>[memaccess/linux] Fix typo<commit_after>package memaccess

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

type process struct {
	pid          uint
	mapsFilepath string
	memFilepath  string
}

func newProcessMemoryReaderImpl(pid uint) (process, error, []error) {
	var result process
	result.pid = pid
	result.mapsFilepath = filepath.Join("/proc", fmt.Sprintf("%d", pid), "maps")
	result.memFilepath = filepath.Join("/proc", fmt.Sprintf("%d", pid), "mem")
	softerrors := make([]error, 0)

	// trying to open the maps file, only to see if we have enought privileges
	f, harderror := os.Open(result.mapsFilepath)
	if harderror != nil {
		return process{}, harderror, softerrors
	}
	f.Close()

	return result, nil, softerrors
}

func (p process) Close() (error, []error) {
	return nil, make([]error, 0)
}

func (p process) NextReadableMemoryRegion(address uintptr) (region MemoryRegion, harderror error, softerrors []error) {
	softerrors = make([]error, 0)

	mapsFile, harderror := os.Open(p.mapsFilepath)
	if harderror != nil {
		return
	}
	defer mapsFile.Close()

	region = MemoryRegion{}
	scanner := bufio.NewScanner(mapsFile)
	splitBySpacesRegexp := regexp.MustCompile("\\s+")

	for scanner.Scan() {
		line := scanner.Text()
		items := splitBySpacesRegexp.Split(line, -1)

		if len(items) != 6 {
			return region, fmt.Errorf("Unrecognised maps line: %s", line), softerrors
		}

		start, end, err := parseMemoryLimits(items[0])
		if err != nil {
			return region, err, softerrors
		}

		if end <= address {
			continue
		}

		// Skip vsyscall as it can't be read. It's a special page mapped by the kernel to accelerate some syscalls.
		if items[5] == "[vsyscall]" {
			continue
		}

		// Check if memory is unreadable
		if items[1][0] == '-' {

			// If we were already reading a region this will just finish it. We only report the softerror when we
			// were actually trying to read it.
			if region.Address != 0 {
				return region, nil, softerrors
			}

			softerrors = append(softerrors, fmt.Errorf("Unreadable memory %s", items[0]))
			continue
		}

		size := uint(end - start)

		// Begenning of a region
		if region.Address == 0 {
			region = MemoryRegion{Address: start, Size: size}
			continue
		}

		// Continuation of a region
		if region.Address+uintptr(region.Size) == start {
			region.Size += size
			continue
		}

		// This map is outside the current region, so we are ready
		return region, nil, softerrors
	}

	// No region left
	if err := scanner.Err(); err != nil {
		return region, err, softerrors
	}

	return NoRegionAvailable, nil, softerrors
}

func (p process) CopyMemory(address uintptr, buffer []byte) (error, []error) {
	mem, harderror := os.Open(p.memFilepath)
	softerrors := make([]error, 0)
	if harderror != nil {
		harderror := fmt.Errorf("Error while reading %d bytes starting at %x: %s", len(buffer), address, harderror)
		return harderror, softerrors
	}
	defer mem.Close()

	bytes_read, harderror := mem.ReadAt(buffer, int64(address))
	if harderror != nil {
		harderror := fmt.Errorf("Error while reading %d bytes starting at %x: %s", len(buffer), address, harderror)
		return harderror, softerrors
	}

	if bytes_read != len(buffer) {
		return fmt.Errorf("Could not read the entire buffer"), softerrors
	}

	return nil, softerrors
}

//Parses the memory limits of a mapping as found in /proc/PID/maps
func parseMemoryLimits(limits string) (start uintptr, end uintptr, err error) {
	fields := strings.Split(limits, "-")
	start64, err := strconv.ParseUint(fields[0], 16, 64)
	if err != nil {
		return 0, 0, err
	}
	start = uintptr(start64)

	end64, err := strconv.ParseUint(fields[1], 16, 64)
	if err != nil {
		return 0, 0, err
	}
	end = uintptr(end64)

	return
}
<|endoftext|>