<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package syslog

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"
	"text/template"
)

// tagOffset represents the substring start value for the tag to return
// the logfileName value from the syslogtag.  Substrings in syslog are
// indexed from 1, hence the + 1.
const tagOffset = len("juju-") + 1

// The rsyslog conf for state server nodes.
// Messages are gathered from other nodes and accumulated in an all-machines.log file.
//
// The apparmor profile is quite strict about where rsyslog can write files.
// Instead of poking with the profile, the local provider now logs to
// {{logDir}}-{{user}}-{{env name}}/all-machines.log, and a symlink is made
// in the local provider log dir to point to that file. The file is also
// created with 0644 so the user can read it without poking permissions. By
// default rsyslog creates files with 0644, but in the ubuntu package, the
// setting is changed to 0640, which means normal users can't read the log
// file. Using a new action directive (new as in not-legacy), we can specify
// the file create mode so it doesn't use the default.
//
// I would dearly love to write the filtering action as follows to avoid setting
// and resetting the global $FileCreateMode, but alas, precise doesn't support it
//
// if $syslogtag startswith "juju{{namespace}}-" then
//   action(type="omfile"
//          File="{{logDir}}{{namespace}}/all-machines.log"
//          Template="JujuLogFormat{{namespace}}"
//          FileCreateMode="0644")
// & stop
//
// Instead we need to mess with the global FileCreateMode.  We set it back
// to the ubuntu default after defining our rule.
const stateServerRsyslogTemplate = `
$ModLoad imfile

# Messages received from remote rsyslog machines have messages prefixed with a space,
# so add one in for local messages too if needed.
$template JujuLogFormat{{namespace}},"%syslogtag:{{tagStart}}:$%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"

$template LongTagForwardFormat,"<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%"

$RuleSet local
:syslogtag, startswith, "juju{{namespace}}-" {{range $i, $bootstrapIP := bootstrapHosts}}{{if $i}} & {{end}}@@{{$bootstrapIP}}:{{portNumber}}{{end}};LongTagForwardFormat
$FileCreateMode 0644
:syslogtag, startswith, "juju{{namespace}}-" {{logDir}}/all-machines.log;JujuLogFormat{{namespace}}
& ~
$FileCreateMode 0640

$RuleSet remote
$FileCreateMode 0644
:syslogtag, startswith, "juju{{namespace}}-" {{logDir}}/all-machines.log;JujuLogFormat{{namespace}}
& ~
$FileCreateMode 0640

$InputFilePersistStateInterval 50
$InputFilePollInterval 5
$InputFileName {{logfilePath}}
$InputFileTag juju{{namespace}}-{{logfileName}}:
$InputFileStateFile {{logfileName}}{{namespace}}
$InputRunFileMonitor
$DefaultRuleset local

$ModLoad imtcp
$DefaultNetstreamDriver gtls
$DefaultNetstreamDriverCAFile {{tlsCACertPath}}
$DefaultNetstreamDriverCertFile {{tlsCertPath}}
$DefaultNetstreamDriverKeyFile {{tlsKeyPath}}
$InputTCPServerStreamDriverAuthMode anon
$InputTCPServerStreamDriverMode 1 # run driver in TLS-only mode

$InputTCPServerBindRuleset remote
$InputTCPServerRun {{portNumber}}
`

// The rsyslog conf for non-state server nodes.
// Messages are forwarded to the state server node.
const nodeRsyslogTemplate = `
$ModLoad imfile

# Enable reliable forwarding.
$ActionQueueType LinkedList
$ActionQueueFileName {{logfileName}}{{namespace}}
$ActionResumeRetryCount -1
$ActionQueueSaveOnShutdown on

$InputFilePersistStateInterval 50
$InputFilePollInterval 5
$InputFileName {{logfilePath}}
$InputFileTag juju{{namespace}}-{{logfileName}}:
$InputFileStateFile {{logfileName}}{{namespace}}
$InputRunFileMonitor

$DefaultNetstreamDriver gtls
$DefaultNetstreamDriverCAFile {{tlsCACertPath}}
$ActionSendStreamDriverAuthMode anon
$ActionSendStreamDriverMode 1 # run driver in TLS-only mode

$template LongTagForwardFormat,"<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%"

:syslogtag, startswith, "juju{{namespace}}-" {{range $i, $bootstrapIP := bootstrapHosts}}{{if $i}} & {{end}}@@{{$bootstrapIP}}:{{portNumber}};LongTagForwardFormat{{end}}
& ~
`

// nodeRsyslogTemplateTLSHeader is prepended to
// nodeRsyslogTemplate if TLS is to be used.
const nodeRsyslogTemplateTLSHeader = `
`

const (
	defaultConfigDir          = "/etc/rsyslog.d"
	defaultCACertFileName     = "ca-cert.pem"
	defaultServerCertFileName = "rsyslog-cert.pem"
	defaultServerKeyFileName  = "rsyslog-key.pem"
)

// SyslogConfigRenderer instances are used to generate a rsyslog conf file.
type SyslogConfigRenderer interface {
	Render() ([]byte, error)
}

// SyslogConfig provides a means to configure and generate rsyslog conf files for
// the state server nodes and unit nodes.
// rsyslog is configured to tail the specified log file.
type SyslogConfig struct {
	// the template representing the config file contents.
	configTemplate string
	// the directory where the config file is written.
	ConfigDir string
	// the config file name.
	ConfigFileName string
	// the name of the log file to tail.
	LogFileName string
	// the addresses of the state server to which messages should be forwarded.
	StateServerAddresses []string
	// CA certificate file name.
	CACertFileName string
	// Server certificate file name.
	ServerCertFileName string
	// Server private key file name.
	ServerKeyFileName string
	// the port number for the listener
	Port int
	// the directory for the logfiles
	LogDir string
	// namespace is used when there are multiple environments on one machine
	Namespace string
}

// NewForwardConfig creates a SyslogConfig instance used on unit nodes to forward log entries
// to the state server nodes.
func NewForwardConfig(logFile, logDir string, port int, namespace string, stateServerAddresses []string) *SyslogConfig {
	conf := &SyslogConfig{
		configTemplate:       nodeRsyslogTemplate,
		StateServerAddresses: stateServerAddresses,
		LogFileName:          logFile,
		Port:                 port,
		LogDir:               logDir,
	}
	if namespace != "" {
		conf.Namespace = "-" + namespace
	}
	return conf
}

// NewAccumulateConfig creates a SyslogConfig instance used to accumulate log entries from the
// various unit nodes.
func NewAccumulateConfig(logFile, logDir string, port int, namespace string, stateServerAddresses []string) *SyslogConfig {
	conf := &SyslogConfig{
		configTemplate:       stateServerRsyslogTemplate,
		LogFileName:          logFile,
		Port:                 port,
		LogDir:               logDir,
		StateServerAddresses: stateServerAddresses,
	}
	if namespace != "" {
		conf.Namespace = "-" + namespace
	}
	return conf
}

func either(a, b string) string {
	if a != "" {
		return a
	}
	return b
}

func (slConfig *SyslogConfig) ConfigFilePath() string {
	dir := either(slConfig.ConfigDir, defaultConfigDir)
	return filepath.Join(dir, slConfig.ConfigFileName)
}

func (slConfig *SyslogConfig) CACertPath() string {
	filename := either(slConfig.CACertFileName, defaultCACertFileName)
	return filepath.Join(slConfig.LogDir, filename)
}

func (slConfig *SyslogConfig) ServerCertPath() string {
	filename := either(slConfig.ServerCertFileName, defaultServerCertFileName)
	return filepath.Join(slConfig.LogDir, filename)
}

func (slConfig *SyslogConfig) ServerKeyPath() string {
	filename := either(slConfig.ServerCertFileName, defaultServerKeyFileName)
	return filepath.Join(slConfig.LogDir, filename)
}

// Render generates the rsyslog config.
func (slConfig *SyslogConfig) Render() ([]byte, error) {
	var bootstrapHosts = func() []string {
		var hosts []string
		for _, addr := range slConfig.StateServerAddresses {
			parts := strings.Split(addr, ":")
			hosts = append(hosts, parts[0])
		}
		return hosts
	}

	var logFilePath = func() string {
		return fmt.Sprintf("%s/%s.log", slConfig.LogDir, slConfig.LogFileName)
	}

	t := template.New("")
	t.Funcs(template.FuncMap{
		"logfileName":    func() string { return slConfig.LogFileName },
		"bootstrapHosts": bootstrapHosts,
		"logfilePath":    logFilePath,
		"portNumber":     func() int { return slConfig.Port },
		"logDir":         func() string { return slConfig.LogDir },
		"namespace":      func() string { return slConfig.Namespace },
		"tagStart":       func() int { return tagOffset + len(slConfig.Namespace) },
		"tlsCACertPath":  slConfig.CACertPath,
		"tlsCertPath":    slConfig.ServerCertPath,
		"tlsKeyPath":     slConfig.ServerKeyPath,
	})

	// Process the rsyslog config template and echo to the conf file.
	p, err := t.Parse(slConfig.configTemplate)
	if err != nil {
		return nil, err
	}
	var confBuf bytes.Buffer
	if err := p.Execute(&confBuf, nil); err != nil {
		return nil, err
	}
	return confBuf.Bytes(), nil
}

// Write generates and writes the rsyslog config.
func (slConfig *SyslogConfig) Write() error {
	data, err := slConfig.Render()
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(slConfig.ConfigFilePath(), data, 0644)
	return err
}
<commit_msg>Use corrected template for state server rsyslog config<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package syslog

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"
	"text/template"
)

// tagOffset represents the substring start value for the tag to return
// the logfileName value from the syslogtag.  Substrings in syslog are
// indexed from 1, hence the + 1.
const tagOffset = len("juju-") + 1

// The rsyslog conf for state server nodes.
// Messages are gathered from other nodes and accumulated in an all-machines.log file.
//
// The apparmor profile is quite strict about where rsyslog can write files.
// Instead of poking with the profile, the local provider now logs to
// {{logDir}}-{{user}}-{{env name}}/all-machines.log, and a symlink is made
// in the local provider log dir to point to that file. The file is also
// created with 0644 so the user can read it without poking permissions. By
// default rsyslog creates files with 0644, but in the ubuntu package, the
// setting is changed to 0640, which means normal users can't read the log
// file. Using a new action directive (new as in not-legacy), we can specify
// the file create mode so it doesn't use the default.
//
// I would dearly love to write the filtering action as follows to avoid setting
// and resetting the global $FileCreateMode, but alas, precise doesn't support it
//
// if $syslogtag startswith "juju{{namespace}}-" then
//   action(type="omfile"
//          File="{{logDir}}{{namespace}}/all-machines.log"
//          Template="JujuLogFormat{{namespace}}"
//          FileCreateMode="0644")
// & stop
//
// Instead we need to mess with the global FileCreateMode.  We set it back
// to the ubuntu default after defining our rule.
const stateServerRsyslogTemplate = `
$ModLoad imfile

# Messages received from remote rsyslog machines have messages prefixed with a space,
# so add one in for local messages too if needed.
$template JujuLogFormat{{namespace}},"%syslogtag:{{tagStart}}:$%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"

$template LongTagForwardFormat,"<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%"

$RuleSet local
:syslogtag, startswith, "juju{{namespace}}-" {{range $i, $bootstrapIP := bootstrapHosts}}{{if $i}} & {{end}}@@{{$bootstrapIP}}:{{portNumber}};LongTagForwardFormat{{end}}
$FileCreateMode 0644
:syslogtag, startswith, "juju{{namespace}}-" {{logDir}}/all-machines.log;JujuLogFormat{{namespace}}
& ~
$FileCreateMode 0640

$RuleSet remote
$FileCreateMode 0644
:syslogtag, startswith, "juju{{namespace}}-" {{logDir}}/all-machines.log;JujuLogFormat{{namespace}}
& ~
$FileCreateMode 0640

$InputFilePersistStateInterval 50
$InputFilePollInterval 5
$InputFileName {{logfilePath}}
$InputFileTag juju{{namespace}}-{{logfileName}}:
$InputFileStateFile {{logfileName}}{{namespace}}
$InputRunFileMonitor
$DefaultRuleset local

$ModLoad imtcp
$DefaultNetstreamDriver gtls
$DefaultNetstreamDriverCAFile {{tlsCACertPath}}
$DefaultNetstreamDriverCertFile {{tlsCertPath}}
$DefaultNetstreamDriverKeyFile {{tlsKeyPath}}
$InputTCPServerStreamDriverAuthMode anon
$InputTCPServerStreamDriverMode 1 # run driver in TLS-only mode

$InputTCPServerBindRuleset remote
$InputTCPServerRun {{portNumber}}
`

// The rsyslog conf for non-state server nodes.
// Messages are forwarded to the state server node.
const nodeRsyslogTemplate = `
$ModLoad imfile

# Enable reliable forwarding.
$ActionQueueType LinkedList
$ActionQueueFileName {{logfileName}}{{namespace}}
$ActionResumeRetryCount -1
$ActionQueueSaveOnShutdown on

$InputFilePersistStateInterval 50
$InputFilePollInterval 5
$InputFileName {{logfilePath}}
$InputFileTag juju{{namespace}}-{{logfileName}}:
$InputFileStateFile {{logfileName}}{{namespace}}
$InputRunFileMonitor

$DefaultNetstreamDriver gtls
$DefaultNetstreamDriverCAFile {{tlsCACertPath}}
$ActionSendStreamDriverAuthMode anon
$ActionSendStreamDriverMode 1 # run driver in TLS-only mode

$template LongTagForwardFormat,"<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%"

:syslogtag, startswith, "juju{{namespace}}-" {{range $i, $bootstrapIP := bootstrapHosts}}{{if $i}} & {{end}}@@{{$bootstrapIP}}:{{portNumber}};LongTagForwardFormat{{end}}
& ~
`

// nodeRsyslogTemplateTLSHeader is prepended to
// nodeRsyslogTemplate if TLS is to be used.
const nodeRsyslogTemplateTLSHeader = `
`

const (
	defaultConfigDir          = "/etc/rsyslog.d"
	defaultCACertFileName     = "ca-cert.pem"
	defaultServerCertFileName = "rsyslog-cert.pem"
	defaultServerKeyFileName  = "rsyslog-key.pem"
)

// SyslogConfigRenderer instances are used to generate a rsyslog conf file.
type SyslogConfigRenderer interface {
	Render() ([]byte, error)
}

// SyslogConfig provides a means to configure and generate rsyslog conf files for
// the state server nodes and unit nodes.
// rsyslog is configured to tail the specified log file.
type SyslogConfig struct {
	// the template representing the config file contents.
	configTemplate string
	// the directory where the config file is written.
	ConfigDir string
	// the config file name.
	ConfigFileName string
	// the name of the log file to tail.
	LogFileName string
	// the addresses of the state server to which messages should be forwarded.
	StateServerAddresses []string
	// CA certificate file name.
	CACertFileName string
	// Server certificate file name.
	ServerCertFileName string
	// Server private key file name.
	ServerKeyFileName string
	// the port number for the listener
	Port int
	// the directory for the logfiles
	LogDir string
	// namespace is used when there are multiple environments on one machine
	Namespace string
}

// NewForwardConfig creates a SyslogConfig instance used on unit nodes to forward log entries
// to the state server nodes.
func NewForwardConfig(logFile, logDir string, port int, namespace string, stateServerAddresses []string) *SyslogConfig {
	conf := &SyslogConfig{
		configTemplate:       nodeRsyslogTemplate,
		StateServerAddresses: stateServerAddresses,
		LogFileName:          logFile,
		Port:                 port,
		LogDir:               logDir,
	}
	if namespace != "" {
		conf.Namespace = "-" + namespace
	}
	return conf
}

// NewAccumulateConfig creates a SyslogConfig instance used to accumulate log entries from the
// various unit nodes.
func NewAccumulateConfig(logFile, logDir string, port int, namespace string, stateServerAddresses []string) *SyslogConfig {
	conf := &SyslogConfig{
		configTemplate:       stateServerRsyslogTemplate,
		LogFileName:          logFile,
		Port:                 port,
		LogDir:               logDir,
		StateServerAddresses: stateServerAddresses,
	}
	if namespace != "" {
		conf.Namespace = "-" + namespace
	}
	return conf
}

func either(a, b string) string {
	if a != "" {
		return a
	}
	return b
}

func (slConfig *SyslogConfig) ConfigFilePath() string {
	dir := either(slConfig.ConfigDir, defaultConfigDir)
	return filepath.Join(dir, slConfig.ConfigFileName)
}

func (slConfig *SyslogConfig) CACertPath() string {
	filename := either(slConfig.CACertFileName, defaultCACertFileName)
	return filepath.Join(slConfig.LogDir, filename)
}

func (slConfig *SyslogConfig) ServerCertPath() string {
	filename := either(slConfig.ServerCertFileName, defaultServerCertFileName)
	return filepath.Join(slConfig.LogDir, filename)
}

func (slConfig *SyslogConfig) ServerKeyPath() string {
	filename := either(slConfig.ServerCertFileName, defaultServerKeyFileName)
	return filepath.Join(slConfig.LogDir, filename)
}

// Render generates the rsyslog config.
func (slConfig *SyslogConfig) Render() ([]byte, error) {
	var bootstrapHosts = func() []string {
		var hosts []string
		for _, addr := range slConfig.StateServerAddresses {
			parts := strings.Split(addr, ":")
			hosts = append(hosts, parts[0])
		}
		return hosts
	}

	var logFilePath = func() string {
		return fmt.Sprintf("%s/%s.log", slConfig.LogDir, slConfig.LogFileName)
	}

	t := template.New("")
	t.Funcs(template.FuncMap{
		"logfileName":    func() string { return slConfig.LogFileName },
		"bootstrapHosts": bootstrapHosts,
		"logfilePath":    logFilePath,
		"portNumber":     func() int { return slConfig.Port },
		"logDir":         func() string { return slConfig.LogDir },
		"namespace":      func() string { return slConfig.Namespace },
		"tagStart":       func() int { return tagOffset + len(slConfig.Namespace) },
		"tlsCACertPath":  slConfig.CACertPath,
		"tlsCertPath":    slConfig.ServerCertPath,
		"tlsKeyPath":     slConfig.ServerKeyPath,
	})

	// Process the rsyslog config template and echo to the conf file.
	p, err := t.Parse(slConfig.configTemplate)
	if err != nil {
		return nil, err
	}
	var confBuf bytes.Buffer
	if err := p.Execute(&confBuf, nil); err != nil {
		return nil, err
	}
	return confBuf.Bytes(), nil
}

// Write generates and writes the rsyslog config.
func (slConfig *SyslogConfig) Write() error {
	data, err := slConfig.Render()
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(slConfig.ConfigFilePath(), data, 0644)
	return err
}
<|endoftext|>


<commit_before>package main

import (
  "strings"
  "log"
  // "fmt"
//  "regexp"
  "encoding/json"
  "os"
  "strconv"
  "github.com/lookify/town/cluster"
  dockerapi "github.com/fsouza/go-dockerclient"
)

const DEFAULT_ENDPOINT = "unix:///var/run/docker.sock"

type Town struct {
  cluster *cluster.Cluster
  docker *dockerapi.Client

//  containers []*dockerapi.Container
}

func NewTown() *Town {
  return &Town{
    cluster: nil,
    docker: nil,
    // containers: []*dockerapi.Container{}
  }
}

func (t *Town) ReadFile() {
  t.cluster = cluster.NewCluster()
  t.cluster.ReadFile()
}

func (t *Town) Connect() {
  envHost := os.Getenv("DOCKER_HOST")
  endpoint := DEFAULT_ENDPOINT
  if (envHost != "") {
    endpoint = envHost;
  }
  log.Println("Using Docker API endpont: ", endpoint)
  docker, err := dockerapi.NewClient( endpoint )
  if err != nil {
    log.Println("Can't connect to the docker")
  }
  t.docker = docker
}

func (t *Town) Provision(checkChanged bool) {
  allContainers, err := t.docker.ListContainers(dockerapi.ListContainersOptions{
    All: true,
  })
  for _, listing := range allContainers {
    container, err := t.docker.InspectContainer(listing.ID)
    if err == nil {
      name := container.Name[1:]
      node, index := t.cluster.FindNodeByName(name)
      if node != nil && index > 0 {
        if node.Container.Exist == nil {
          node.Container.Exist = []cluster.ExistContainer{}
        }
        runningContainer := cluster.NewExistContainer(listing.ID, name, index, container.State.Running)
        if checkChanged {
          node.Container.Changed = t.isChangedImage(node, container)
        }
        node.Container.Exist = append(node.Container.Exist, runningContainer);
      }
    } else {
      log.Println("[ERROR] Unable to inspect container:", listing.ID[:12], err)
    }
  }

  if checkChanged {
    t.cluster.AddChangeDependant()
  }
}

/**
 * Check node and running container for changes.
 * TODO: add cache to docker call.
 **/
func (t *Town) isChangedImage(node *cluster.Node, container *dockerapi.Container) bool {
  var imageName = container.Image
  image , error := t.docker.InspectImage(imageName)
  if error == nil {
    secondImage , secondError := t.docker.InspectImage(node.Container.Image)
    if secondError == nil {
      return secondImage.Created.After(image.Created)
    } else {
      log.Println("[ERROR] Could not inspect image ", node.Container.Name)
    }
  } else {
    log.Println("[ERROR] Could not inspect image ", imageName)
  }
  return false
}

func (t *Town) StopContainers(checkChanged bool) {
  log.Println("Stop...")
  //for node := range t.cluster.nodes {
  for i := len(t.cluster.Nodes) - 1; i >= 0; i-- {
    node := t.cluster.Nodes[i]
    if (!checkChanged || node.Container.Changed) && len(node.Container.Exist) > 0 {
      for _, container := range node.Container.Exist {
        if container.Running {
          err := t.docker.StopContainer(container.ID, 10)
          if err == nil {
            log.Println("   -  ", container.Name)
          } else {
            log.Println("   -  ", container.Name, " failed ", err)
          }
        }
      }
    }
  }
  log.Println("=============================")
}

func (t *Town) RemoveContainers(checkChanged bool) {
  log.Println("Remove...")
  //for node := range t.cluster.nodes {
  for i := len(t.cluster.Nodes) - 1; i >= 0; i-- {
    node := t.cluster.Nodes[i]
    if (!checkChanged || node.Container.Changed) && len(node.Container.Exist) > 0 {
      for _, container := range node.Container.Exist {
        err := t.docker.RemoveContainer(dockerapi.RemoveContainerOptions{
          ID: container.ID,
          RemoveVolumes: false,
        })
        if err == nil {
          log.Println("   -  ", container.Name)
        } else {
          log.Println("   -  ", container.Name, " failed ", err)
        }
      }
    }
  }
  log.Println("=============================")
}

func (t *Town) createContainer(node *cluster.Node, index int) (string, string) {
  containerName := node.Container.Name + "-" + strconv.Itoa(index)

  log.Println("   -  ", containerName)

  node.Container.Hostname = containerName // ?? Help !!!!

  env :=  make([]string, 0, cap(node.Container.Environment))
  for _, e := range node.Container.Environment {
    env = append(env, t.exec(e, index))
  }

  volumes := make(map[string]struct{})
  binds := make([]string, 0, cap(node.Container.Volumes))
  if len(node.Container.Volumes) > 0 {
    for _, volume := range node.Container.Volumes {
      volume = t.exec(volume, index)
      vol := strings.Split(volume, ":")
      if len(vol) > 1 {
        volumes[vol[1]] = struct{}{}
      } else {
        volumes[vol[0]] = struct{}{}
      }
      binds = append(binds, volume)
    }
  }

  dockerConfig := dockerapi.Config{
      Image: node.Container.Image,
      Hostname: node.Container.Hostname,
      PortSpecs: node.Container.Ports,
      Env: env,
      Volumes: volumes,

      AttachStdout: false,
      AttachStdin: false,
      AttachStderr: false,

      Tty: false,

      //Cmd: []
  }

  if len(node.Container.Command) > 0 {
    cmd := t.exec(node.Container.Command, index)
    dockerConfig.Cmd = []string{ cmd }
  }

  // just info
  //for _, l := range node.status.links {
  //  log.Println("     * ", l)
  //}

  // create links
  links := t.cluster.GetLinks(node)

  portBindings := map[dockerapi.Port][]dockerapi.PortBinding{}
  // create ports
  for _, ports := range node.Container.Ports {

    port := strings.Split(ports, ":")
    var p dockerapi.Port

    if len(port) > 1 {
      p = dockerapi.Port(port[1] + "/tcp")
    } else {
      p = dockerapi.Port(port[0] + "/tcp")
    }

    if portBindings[p] == nil {
      portBindings[p] = [] dockerapi.PortBinding {}
    }

    portBindings[p] = append(portBindings[p], dockerapi.PortBinding{
      HostIP: "",
      HostPort: port[0],
    })
  }

  hostConfig := dockerapi.HostConfig{
    Binds: binds,
    Links: links, //, [],
    PortBindings: portBindings,
    NetworkMode: "bridge",
    PublishAllPorts: false,
    Privileged: node.Container.Privileged,
  }

  opts := dockerapi.CreateContainerOptions{Name: containerName, Config: &dockerConfig, HostConfig: &hostConfig}
  container, err := t.docker.CreateContainer(opts)
  if err == nil {
    retry := 5
    for retry > 0 {
      error := t.docker.StartContainer(container.ID, &hostConfig)
      if error != nil {
        // log.Println("start error: ", error);

        out, err := json.Marshal(container)
        if err != nil {
            panic (err)
        }
        // fmt.Println(string(out))

        retry--;
        if retry == 0 {
          log.Println(" Strat failed after 5 retries: ", string(out))
        }
        // log.Println("retry: ", retry);
      } else {
        inspect, inspectError := t.docker.InspectContainer(container.ID)
        if inspectError == nil {
          //links = append(links, inspect.NetworkSettings.IPAddress + "  " + containerName)
          //ids = append(ids, container.ID)
          return container.ID, inspect.NetworkSettings.IPAddress + "  " + containerName
        } else {
          log.Println("Inpect ", container.ID, " error ", inspectError)
        }
        //retry = 0
        break;
      }
    }
  } else {
    log.Println("error: ", err);
  }

  return nil, nil
}

func (t *Town) CreateContainers(checkChanged bool) {
  log.Println("Create...")
  for _, node := range t.cluster.Nodes {

    if !checkChanged || node.Container.Changed {
      ids := make([]string, 0, node.Container.Scale )

      hosts := make([]string, 0, node.Container.Scale)

      for i := 1; i <= node.Container.Scale; i++ {
        log.Println(node.Container.Name, "  image: ", node.Container.Image)
        id, host = t.CreateContainer(node, i)
        ids = append(ids, id)
        hosts = append(links, host)
      }

      if len(ids) > 1 {
        for index, id := range ids {
          var buffer bytes.Buffer

          buffer.WriteString("echo -e '")
          for i := 0; i < len(hosts); i++ {
            if i != index {
              buffer.WriteString("\n")
              buffer.WriteString(hosts[i])
            }
          }
          buffer.WriteString("' >> /etc/hosts; touch /tmp/host-generated")

          config := dockerapi.CreateExecOptions{
            Container:    id,
            AttachStdin:  true,
            AttachStdout: true,
            AttachStderr: false,
            Tty:          false,
            Cmd:          []string{"bash", "-c", buffer.String()},
          }
          execObj, err := t.docker.CreateExec(config)
          if err == nil {
            config := dockerapi.StartExecOptions{
              Detach: true,
            }
            err := t.docker.StartExec(execObj.ID, config)
            if err != nil {
              log.Println("Start exec failed ", id, " error: ", err)
            }
          } else {
            log.Println("Create exec failed ", id, " error: ", err)
          }
        }
      }
    }
  }
}

func (t *Town) exec(text string, scale int) string {
  replace := strings.Replace(text, "${SCALE_NUM}", strconv.Itoa(scale), -1)
  match := SCALE_NUM_REG.FindAllStringSubmatch(replace, -1)
  hostMatch := HOSTS_REG.FindAllStringSubmatch(replace, -1)
  if len(match) > 0 {
    if len(match[0]) > 1 {
      nums := strings.Split(match[0][1], ",")
      if len(nums) > (scale - 1) {
        replace = strings.Replace(replace, match[0][0], nums[scale - 1], -1)
      }
    }
  }
  if len(hostMatch) > 0 {
    if len(hostMatch[0]) > 1 {
      //nums := strings.Split(, ",")
      name := strings.ToLower(hostMatch[0][1])
      node := t.cluster.graph.FindNodeByID(name)

      var buffer bytes.Buffer
      for i := 1; i <= node.Container.Scale; i++ {
        buffer.WriteString(name)
        buffer.WriteString("-")
        buffer.WriteString(strconv.Itoa( i ))
        if i != node.Container.Scale {
          buffer.WriteString(",")
        }
      }
      replace = strings.Replace(replace, hostMatch[0][0], buffer.String(), -1)
    }
  }
  return replace
}
<commit_msg>fix CreateContainer<commit_after>package main

import (
  "strings"
  "log"
  // "fmt"
//  "regexp"
  "encoding/json"
  "os"
  "strconv"
  "github.com/lookify/town/cluster"
  dockerapi "github.com/fsouza/go-dockerclient"
)

const DEFAULT_ENDPOINT = "unix:///var/run/docker.sock"

type Town struct {
  cluster *cluster.Cluster
  docker *dockerapi.Client

//  containers []*dockerapi.Container
}

func NewTown() *Town {
  return &Town{
    cluster: nil,
    docker: nil,
    // containers: []*dockerapi.Container{}
  }
}

func (t *Town) ReadFile() {
  t.cluster = cluster.NewCluster()
  t.cluster.ReadFile()
}

func (t *Town) Connect() {
  envHost := os.Getenv("DOCKER_HOST")
  endpoint := DEFAULT_ENDPOINT
  if (envHost != "") {
    endpoint = envHost;
  }
  log.Println("Using Docker API endpont: ", endpoint)
  docker, err := dockerapi.NewClient( endpoint )
  if err != nil {
    log.Println("Can't connect to the docker")
  }
  t.docker = docker
}

func (t *Town) Provision(checkChanged bool) {
  allContainers, err := t.docker.ListContainers(dockerapi.ListContainersOptions{
    All: true,
  })
  for _, listing := range allContainers {
    container, err := t.docker.InspectContainer(listing.ID)
    if err == nil {
      name := container.Name[1:]
      node, index := t.cluster.FindNodeByName(name)
      if node != nil && index > 0 {
        if node.Container.Exist == nil {
          node.Container.Exist = []cluster.ExistContainer{}
        }
        runningContainer := cluster.NewExistContainer(listing.ID, name, index, container.State.Running)
        if checkChanged {
          node.Container.Changed = t.isChangedImage(node, container)
        }
        node.Container.Exist = append(node.Container.Exist, runningContainer);
      }
    } else {
      log.Println("[ERROR] Unable to inspect container:", listing.ID[:12], err)
    }
  }

  if checkChanged {
    t.cluster.AddChangeDependant()
  }
}

/**
 * Check node and running container for changes.
 * TODO: add cache to docker call.
 **/
func (t *Town) isChangedImage(node *cluster.Node, container *dockerapi.Container) bool {
  var imageName = container.Image
  image , error := t.docker.InspectImage(imageName)
  if error == nil {
    secondImage , secondError := t.docker.InspectImage(node.Container.Image)
    if secondError == nil {
      return secondImage.Created.After(image.Created)
    } else {
      log.Println("[ERROR] Could not inspect image ", node.Container.Name)
    }
  } else {
    log.Println("[ERROR] Could not inspect image ", imageName)
  }
  return false
}

func (t *Town) StopContainers(checkChanged bool) {
  log.Println("Stop...")
  //for node := range t.cluster.nodes {
  for i := len(t.cluster.Nodes) - 1; i >= 0; i-- {
    node := t.cluster.Nodes[i]
    if (!checkChanged || node.Container.Changed) && len(node.Container.Exist) > 0 {
      for _, container := range node.Container.Exist {
        if container.Running {
          err := t.docker.StopContainer(container.ID, 10)
          if err == nil {
            log.Println("   -  ", container.Name)
          } else {
            log.Println("   -  ", container.Name, " failed ", err)
          }
        }
      }
    }
  }
  log.Println("=============================")
}

func (t *Town) RemoveContainers(checkChanged bool) {
  log.Println("Remove...")
  //for node := range t.cluster.nodes {
  for i := len(t.cluster.Nodes) - 1; i >= 0; i-- {
    node := t.cluster.Nodes[i]
    if (!checkChanged || node.Container.Changed) && len(node.Container.Exist) > 0 {
      for _, container := range node.Container.Exist {
        err := t.docker.RemoveContainer(dockerapi.RemoveContainerOptions{
          ID: container.ID,
          RemoveVolumes: false,
        })
        if err == nil {
          log.Println("   -  ", container.Name)
        } else {
          log.Println("   -  ", container.Name, " failed ", err)
        }
      }
    }
  }
  log.Println("=============================")
}

func (t *Town) CreateContainer(node *cluster.Node, index int) (*string, *string) {
  containerName := node.Container.Name + "-" + strconv.Itoa(index)

  log.Println("   -  ", containerName)

  node.Container.Hostname = containerName // ?? Help !!!!

  env :=  make([]string, 0, cap(node.Container.Environment))
  for _, e := range node.Container.Environment {
    env = append(env, t.exec(e, index))
  }

  volumes := make(map[string]struct{})
  binds := make([]string, 0, cap(node.Container.Volumes))
  if len(node.Container.Volumes) > 0 {
    for _, volume := range node.Container.Volumes {
      volume = t.exec(volume, index)
      vol := strings.Split(volume, ":")
      if len(vol) > 1 {
        volumes[vol[1]] = struct{}{}
      } else {
        volumes[vol[0]] = struct{}{}
      }
      binds = append(binds, volume)
    }
  }

  dockerConfig := dockerapi.Config{
      Image: node.Container.Image,
      Hostname: node.Container.Hostname,
      PortSpecs: node.Container.Ports,
      Env: env,
      Volumes: volumes,

      AttachStdout: false,
      AttachStdin: false,
      AttachStderr: false,

      Tty: false,

      //Cmd: []
  }

  if len(node.Container.Command) > 0 {
    cmd := t.exec(node.Container.Command, index)
    dockerConfig.Cmd = []string{ cmd }
  }

  // just info
  //for _, l := range node.status.links {
  //  log.Println("     * ", l)
  //}

  // create links
  links := t.cluster.GetLinks(node)

  portBindings := map[dockerapi.Port][]dockerapi.PortBinding{}
  // create ports
  for _, ports := range node.Container.Ports {

    port := strings.Split(ports, ":")
    var p dockerapi.Port

    if len(port) > 1 {
      p = dockerapi.Port(port[1] + "/tcp")
    } else {
      p = dockerapi.Port(port[0] + "/tcp")
    }

    if portBindings[p] == nil {
      portBindings[p] = [] dockerapi.PortBinding {}
    }

    portBindings[p] = append(portBindings[p], dockerapi.PortBinding{
      HostIP: "",
      HostPort: port[0],
    })
  }

  hostConfig := dockerapi.HostConfig{
    Binds: binds,
    Links: links, //, [],
    PortBindings: portBindings,
    NetworkMode: "bridge",
    PublishAllPorts: false,
    Privileged: node.Container.Privileged,
  }

  opts := dockerapi.CreateContainerOptions{Name: containerName, Config: &dockerConfig, HostConfig: &hostConfig}
  container, err := t.docker.CreateContainer(opts)
  if err == nil {
    retry := 5
    for retry > 0 {
      error := t.docker.StartContainer(container.ID, &hostConfig)
      if error != nil {
        // log.Println("start error: ", error);

        out, err := json.Marshal(container)
        if err != nil {
            panic (err)
        }
        // fmt.Println(string(out))

        retry--;
        if retry == 0 {
          log.Println(" Strat failed after 5 retries: ", string(out))
        }
        // log.Println("retry: ", retry);
      } else {
        inspect, inspectError := t.docker.InspectContainer(container.ID)
        if inspectError == nil {
          //links = append(links, inspect.NetworkSettings.IPAddress + "  " + containerName)
          //ids = append(ids, container.ID)
          return container.ID, inspect.NetworkSettings.IPAddress + "  " + containerName
        } else {
          log.Println("Inpect ", container.ID, " error ", inspectError)
        }
        //retry = 0
        break;
      }
    }
  } else {
    log.Println("error: ", err);
  }

  return nil, nil
}

func (t *Town) CreateContainers(checkChanged bool) {
  log.Println("Create...")
  for _, node := range t.cluster.Nodes {

    if !checkChanged || node.Container.Changed {
      ids := make([]string, 0, node.Container.Scale )

      hosts := make([]string, 0, node.Container.Scale)

      for i := 1; i <= node.Container.Scale; i++ {
        log.Println(node.Container.Name, "  image: ", node.Container.Image)
        id, host = t.CreateContainer(node, i)
        ids = append(ids, id)
        hosts = append(links, host)
      }

      if len(ids) > 1 {
        for index, id := range ids {
          var buffer bytes.Buffer

          buffer.WriteString("echo -e '")
          for i := 0; i < len(hosts); i++ {
            if i != index {
              buffer.WriteString("\n")
              buffer.WriteString(hosts[i])
            }
          }
          buffer.WriteString("' >> /etc/hosts; touch /tmp/host-generated")

          config := dockerapi.CreateExecOptions{
            Container:    id,
            AttachStdin:  true,
            AttachStdout: true,
            AttachStderr: false,
            Tty:          false,
            Cmd:          []string{"bash", "-c", buffer.String()},
          }
          execObj, err := t.docker.CreateExec(config)
          if err == nil {
            config := dockerapi.StartExecOptions{
              Detach: true,
            }
            err := t.docker.StartExec(execObj.ID, config)
            if err != nil {
              log.Println("Start exec failed ", id, " error: ", err)
            }
          } else {
            log.Println("Create exec failed ", id, " error: ", err)
          }
        }
      }
    }
  }
}

func (t *Town) exec(text string, scale int) string {
  replace := strings.Replace(text, "${SCALE_NUM}", strconv.Itoa(scale), -1)
  match := SCALE_NUM_REG.FindAllStringSubmatch(replace, -1)
  hostMatch := HOSTS_REG.FindAllStringSubmatch(replace, -1)
  if len(match) > 0 {
    if len(match[0]) > 1 {
      nums := strings.Split(match[0][1], ",")
      if len(nums) > (scale - 1) {
        replace = strings.Replace(replace, match[0][0], nums[scale - 1], -1)
      }
    }
  }
  if len(hostMatch) > 0 {
    if len(hostMatch[0]) > 1 {
      //nums := strings.Split(, ",")
      name := strings.ToLower(hostMatch[0][1])
      node := t.cluster.graph.FindNodeByID(name)

      var buffer bytes.Buffer
      for i := 1; i <= node.Container.Scale; i++ {
        buffer.WriteString(name)
        buffer.WriteString("-")
        buffer.WriteString(strconv.Itoa( i ))
        if i != node.Container.Scale {
          buffer.WriteString(",")
        }
      }
      replace = strings.Replace(replace, hostMatch[0][0], buffer.String(), -1)
    }
  }
  return replace
}
<|endoftext|>


<commit_before>package main

import (
  "os"
  "fmt"
  "context"

  "github.com/kataras/iris"
  "github.com/docker/docker/api/types"
  "github.com/docker/docker/api/types/swarm"
  "github.com/docker/docker/client"
)

type SwarmistService struct{
  swarm.Service
  ReplicasRunning int
}

func main(){
  cli, err := client.NewEnvClient()
  if err != nil {
    panic(err)
  }

  iris.Get("/", func(ctx *iris.Context) {
    requestpath := "client/build/index.html"
    ctx.ServeFile(requestpath, true)
  })

  iris.Get("/static/*file", func(ctx *iris.Context) {
    requestpath := "client/build/static" + ctx.Param("file")
    ctx.ServeFile(requestpath, true)
  })

  iris.Get("/api/services", func(ctx *iris.Context){
    services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{})
    if err != nil {
      fmt.Println(err)
      ctx.EmitError(iris.StatusInternalServerError)
      return
    }

    tasks, err := cli.TaskList(context.Background(), types.TaskListOptions{})
    if err != nil {
      fmt.Println(err)
      ctx.EmitError(iris.StatusInternalServerError)
      return
    }

    swarmistServices := make([]SwarmistService, len(services))
    for i,service := range services {
      swarmistService := SwarmistService{service, 0}

      replicasRunning := 0
      for _,task := range tasks {
        if (task.ServiceID == service.ID && task.Status.State == "running") {
          replicasRunning++
        }
      }
      swarmistService.ReplicasRunning = replicasRunning
      swarmistServices[i] = swarmistService
    }

    ctx.JSON(iris.StatusOK, swarmistServices)
  })

  iris.Get("/api/tasks", func(ctx *iris.Context){
    tasks, err := cli.TaskList(context.Background(), types.TaskListOptions{})
    if err != nil {
      ctx.EmitError(iris.StatusInternalServerError)
      return
    }

    ctx.JSON(iris.StatusOK, tasks)
  })

  listenTo := ":3000"
  if os.Getenv("PORT") != "" {
    listenTo = ":" + os.Getenv("PORT")
  }
  fmt.Println("Listening to ", listenTo)
  iris.Listen(listenTo)
}
<commit_msg>fix: remove redundant port logging<commit_after>package main

import (
  "os"
  "fmt"
  "context"

  "github.com/kataras/iris"
  "github.com/docker/docker/api/types"
  "github.com/docker/docker/api/types/swarm"
  "github.com/docker/docker/client"
)

type SwarmistService struct{
  swarm.Service
  ReplicasRunning int
}

func main(){
  cli, err := client.NewEnvClient()
  if err != nil {
    panic(err)
  }

  iris.Get("/", func(ctx *iris.Context) {
    requestpath := "client/build/index.html"
    ctx.ServeFile(requestpath, true)
  })

  iris.Get("/static/*file", func(ctx *iris.Context) {
    requestpath := "client/build/static" + ctx.Param("file")
    ctx.ServeFile(requestpath, true)
  })

  iris.Get("/api/services", func(ctx *iris.Context){
    services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{})
    if err != nil {
      fmt.Println(err)
      ctx.EmitError(iris.StatusInternalServerError)
      return
    }

    tasks, err := cli.TaskList(context.Background(), types.TaskListOptions{})
    if err != nil {
      fmt.Println(err)
      ctx.EmitError(iris.StatusInternalServerError)
      return
    }

    swarmistServices := make([]SwarmistService, len(services))
    for i,service := range services {
      swarmistService := SwarmistService{service, 0}

      replicasRunning := 0
      for _,task := range tasks {
        if (task.ServiceID == service.ID && task.Status.State == "running") {
          replicasRunning++
        }
      }
      swarmistService.ReplicasRunning = replicasRunning
      swarmistServices[i] = swarmistService
    }

    ctx.JSON(iris.StatusOK, swarmistServices)
  })

  iris.Get("/api/tasks", func(ctx *iris.Context){
    tasks, err := cli.TaskList(context.Background(), types.TaskListOptions{})
    if err != nil {
      ctx.EmitError(iris.StatusInternalServerError)
      return
    }

    ctx.JSON(iris.StatusOK, tasks)
  })

  listenTo := ":3000"
  if os.Getenv("PORT") != "" {
    listenTo = ":" + os.Getenv("PORT")
  }
  iris.Listen(listenTo)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"github.com/jackc/pgx"
	"log"
	"net/http"
	"strings"
	"time"
)

const CitySQL = `SELECT cities.name, cities.alternate_names AS names, cities.country, cities.link,
    cities.coordinate[0] AS latitude, cities.coordinate[1] AS longitude, cities.elevation,
    cities.population,
    jsonb_agg(jsonb_build_object('id', stations.id, 'name', stations.name, 'icao', stations.icao,
    'country', stations.country, 'latitude', stations.coordinate[0],
    'longitude', stations.coordinate[1], 'elevation', stations.elevation)) AS stations
FROM cities
    CROSS JOIN LATERAL (SELECT * FROM stations
            INNER JOIN gsod_availability ON stations.id = gsod_availability.station_id AND gsod_availability.year = $2
                 AND gsod_availability.max_temperature > 350 AND gsod_availability.min_temperature > 350 AND gsod_availability.precipitation > 0
        ORDER BY cities.coordinate <-> stations.coordinate LIMIT 8) stations
    WHERE cities.alternate_names @> ('[{"name": "' || $1 || '"}]')::JSONB
GROUP BY cities.id`

const StationSQL = `SELECT stations.id, stations.name, stations.icao, stations.country,
    stations.coordinate[0] AS latitude, stations.coordinate[1] AS longitude, stations.elevation,
    jsonb_agg(jsonb_build_object(
                  'name', cities.name, 'names', cities.alternate_names, 'country', cities.country,
                  'link', cities.link, 'latitude', cities.coordinate[0], 'longitude', cities.coordinate[1],
                  'elevation', cities.elevation, 'population', cities.population)) AS cities
FROM stations CROSS JOIN LATERAL (
    SELECT * FROM cities ORDER BY cities.coordinate <-> stations.coordinate LIMIT 8
) AS cities
WHERE stations.name = $1 OR stations.id = $1 OR stations.icao = $1
GROUP BY stations.id`

var pgConn *pgx.ConnPool

func makeSlice(fields []pgx.FieldDescription) (result []interface{}) {
	for _, field := range fields {
		switch field.DataTypeName {
		case "text":
			var v string
			result = append(result, &v)
		case "jsonb":
			var v interface{}
			result = append(result, &v)
		case "float4", "float8":
			var v float32
			result = append(result, &v)
		case "int8":
			var v int64
			result = append(result, &v)
		default:
			panic(field.DataTypeName)
		}
	}
	return
}
func queryDB(sql string, args ...interface{}) ([]byte, error) {
	rows, err := pgConn.Query(sql, args...)
	if err != nil {
		return nil, err
	}
	var result []map[string]interface{}
	fields := rows.FieldDescriptions()
	for rows.Next() {
		values := makeSlice(fields)
		err = rows.Scan(values...)
		if err != nil {
			return nil, err
		}
		row := make(map[string]interface{})
		for i, field := range fields {
			row[field.Name] = values[i]
		}
		result = append(result, row)
	}
	return json.Marshal(result)
}

func writeResponse(w http.ResponseWriter, data []byte, err error) {
	if err != nil {
		log.Println(err)
		http.Error(w, "internal error", http.StatusInternalServerError)
		return
	}
	if bytes.Equal(data, []byte("null")) {
		http.Error(w, "not found", http.StatusNotFound)
		return
	}
	_, err = w.Write(data)
	if err != nil {
		log.Println(err)
	}
}
func cityHandler(w http.ResponseWriter, req *http.Request) {
	query := req.URL.Query()
	if len(query["year"]) != 1 || len(query["name"]) != 1 {
		http.Error(w, "invalid argument", http.StatusBadRequest)
		return
	}
	year := query["year"][0]
	name := query["name"][0]
	result, err := queryDB(CitySQL, name, year)
	writeResponse(w, result, err)
}
func stationHandler(w http.ResponseWriter, req *http.Request) {
	query := req.URL.Query()
	if len(query["name"]) != 1 {
		http.Error(w, "invalid argument", http.StatusBadRequest)
		return
	}
	name := query["name"][0]
	result, err := queryDB(StationSQL, name)
	writeResponse(w, result, err)
}

func Handler(w http.ResponseWriter, req *http.Request) {
	start := time.Now()
	defer func() {
		log.Println(req.URL, time.Since(start))
	}()
	path := strings.Trim(req.URL.Path, "/")
	w.Header().Add("Content-Type", "application/json")
	switch path {
	case "city":
		cityHandler(w, req)
	case "station":
		stationHandler(w, req)
	default:
		http.Error(w, "invalid path", http.StatusBadRequest)
	}
}
func connect() *pgx.ConnPool {
	conn, err := pgx.NewConnPool(pgx.ConnPoolConfig{
		ConnConfig: pgx.ConnConfig{
			Database: "days",
		},
		MaxConnections: 16,
	})
	if err != nil {
		panic(err)
	}
	return conn
}

func main() {
	pgConn = connect()
	http.HandleFunc("/", Handler)
	log.Fatal(http.ListenAndServe("127.0.0.1:3130", nil))
}
<commit_msg>in city searching, add distance between city and station<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"github.com/jackc/pgx"
	"log"
	"net/http"
	"strings"
	"time"
)

const CitySQL = `SELECT cities.name, cities.alternate_names AS names, cities.country, cities.link,
    cities.coordinate[0] AS latitude, cities.coordinate[1] AS longitude, cities.elevation, cities.population,
    jsonb_agg(jsonb_build_object('id', stations.id, 'name', stations.name, 'icao', stations.icao,
        'country', stations.country, 'latitude', stations.coordinate[0], 'longitude', stations.coordinate[1],
        'elevation', stations.elevation, 'distance', stations.distance) ORDER BY stations.distance) AS stations
FROM cities
    CROSS JOIN LATERAL (
        SELECT *, cities.coordinate <-> stations.coordinate AS distance FROM stations
            INNER JOIN gsod_availability ON stations.id = gsod_availability.station_id AND gsod_availability.year = $2
                 AND gsod_availability.max_temperature > 350 AND gsod_availability.min_temperature > 350 AND gsod_availability.precipitation > 0
            ORDER BY cities.coordinate <-> stations.coordinate LIMIT 8
        ) AS stations
    WHERE cities.alternate_names @> ('[{"name": "' || $1 || '"}]')::JSONB
GROUP BY cities.id`

const StationSQL = `SELECT stations.id, stations.name, stations.icao, stations.country,
    stations.coordinate[0] AS latitude, stations.coordinate[1] AS longitude, stations.elevation,
    jsonb_agg(jsonb_build_object(
                  'name', cities.name, 'names', cities.alternate_names, 'country', cities.country,
                  'link', cities.link, 'latitude', cities.coordinate[0], 'longitude', cities.coordinate[1],
                  'elevation', cities.elevation, 'population', cities.population)) AS cities
FROM stations CROSS JOIN LATERAL (
    SELECT * FROM cities ORDER BY cities.coordinate <-> stations.coordinate LIMIT 8
) AS cities
WHERE stations.name = $1 OR stations.id = $1 OR stations.icao = $1
GROUP BY stations.id`

var pgConn *pgx.ConnPool

func makeSlice(fields []pgx.FieldDescription) (result []interface{}) {
	for _, field := range fields {
		switch field.DataTypeName {
		case "text":
			var v string
			result = append(result, &v)
		case "jsonb":
			var v interface{}
			result = append(result, &v)
		case "float4", "float8":
			var v float32
			result = append(result, &v)
		case "int8":
			var v int64
			result = append(result, &v)
		default:
			panic(field.DataTypeName)
		}
	}
	return
}
func queryDB(sql string, args ...interface{}) ([]byte, error) {
	rows, err := pgConn.Query(sql, args...)
	if err != nil {
		return nil, err
	}
	var result []map[string]interface{}
	fields := rows.FieldDescriptions()
	for rows.Next() {
		values := makeSlice(fields)
		err = rows.Scan(values...)
		if err != nil {
			return nil, err
		}
		row := make(map[string]interface{})
		for i, field := range fields {
			row[field.Name] = values[i]
		}
		result = append(result, row)
	}
	return json.Marshal(result)
}

func writeResponse(w http.ResponseWriter, data []byte, err error) {
	if err != nil {
		log.Println(err)
		http.Error(w, "internal error", http.StatusInternalServerError)
		return
	}
	if bytes.Equal(data, []byte("null")) {
		http.Error(w, "not found", http.StatusNotFound)
		return
	}
	_, err = w.Write(data)
	if err != nil {
		log.Println(err)
	}
}
func cityHandler(w http.ResponseWriter, req *http.Request) {
	query := req.URL.Query()
	if len(query["year"]) != 1 || len(query["name"]) != 1 {
		http.Error(w, "invalid argument", http.StatusBadRequest)
		return
	}
	year := query["year"][0]
	name := query["name"][0]
	result, err := queryDB(CitySQL, name, year)
	writeResponse(w, result, err)
}
func stationHandler(w http.ResponseWriter, req *http.Request) {
	query := req.URL.Query()
	if len(query["name"]) != 1 {
		http.Error(w, "invalid argument", http.StatusBadRequest)
		return
	}
	name := query["name"][0]
	result, err := queryDB(StationSQL, name)
	writeResponse(w, result, err)
}

func Handler(w http.ResponseWriter, req *http.Request) {
	start := time.Now()
	defer func() {
		log.Println(req.URL, time.Since(start))
	}()
	path := strings.Trim(req.URL.Path, "/")
	w.Header().Add("Content-Type", "application/json")
	switch path {
	case "city":
		cityHandler(w, req)
	case "station":
		stationHandler(w, req)
	default:
		http.Error(w, "invalid path", http.StatusBadRequest)
	}
}
func connect() *pgx.ConnPool {
	conn, err := pgx.NewConnPool(pgx.ConnPoolConfig{
		ConnConfig: pgx.ConnConfig{
			Database: "days",
		},
		MaxConnections: 16,
	})
	if err != nil {
		panic(err)
	}
	return conn
}

func main() {
	pgConn = connect()
	http.HandleFunc("/", Handler)
	log.Fatal(http.ListenAndServe("127.0.0.1:3130", nil))
}
<|endoftext|>


<commit_before>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build ignore

package main

import (
	"errors"
	"image"
	"image/color"
	"image/png"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"text/template"

	"github.com/golang/freetype/truetype"
	"golang.org/x/image/font"

	"github.com/hajimehoshi/ebiten"
	"github.com/hajimehoshi/ebiten/text"
)

const (
	arcadeFontSize = 8
)

var (
	arcadeFont font.Face
)

func init() {
	f, err := os.Open(filepath.Join("..", "..", "resources", "fonts", "arcade_n.ttf"))
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	b, err := ioutil.ReadAll(f)
	if err != nil {
		log.Fatal(err)
	}

	tt, err := truetype.Parse(b)
	if err != nil {
		log.Fatal(err)
	}

	const dpi = 72
	arcadeFont = truetype.NewFace(tt, &truetype.Options{
		Size:    arcadeFontSize,
		DPI:     dpi,
		Hinting: font.HintingFull,
	})
}

var keyboardKeys = [][]string{
	{"Esc", "1", "2", "3", "4", "5", "6", "7", "8", "9", "0", "-", "=", "\\", "`", " "},
	{"Tab", "Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P", "[", "]", "BS"},
	{"Ctrl", "A", "S", "D", "F", "G", "H", "J", "K", "L", ";", "'", "Enter"},
	{"Shift", "Z", "X", "C", "V", "B", "N", "M", ",", ".", "/", " "},
	{" ", "Alt", "Space", " ", " "},
	{},
	{"", "Up", ""},
	{"Left", "Down", "Right"},
}

func keyDisplayNameToKey(name string) ebiten.Key {
	switch name {
	case "Esc":
		return ebiten.KeyEscape
	case "Tab":
		return ebiten.KeyTab
	case "Ctrl":
		return ebiten.KeyControl
	case "Shift":
		return ebiten.KeyShift
	case "Alt":
		return ebiten.KeyAlt
	case "Space":
		return ebiten.KeySpace
	case "Up":
		return ebiten.KeyUp
	case "Left":
		return ebiten.KeyLeft
	case "Down":
		return ebiten.KeyDown
	case "Right":
		return ebiten.KeyRight
	case "BS":
		return ebiten.KeyBackspace
	case "Enter":
		return ebiten.KeyEnter
	case "-":
		return ebiten.KeyMinus
	case "=":
		return ebiten.KeyEqual
	case "\\":
		return ebiten.KeyBackslash
	case "`":
		return ebiten.KeyGraveAccent
	case "[":
		return ebiten.KeyLeftBracket
	case "]":
		return ebiten.KeyRightBracket
	case ";":
		return ebiten.KeySemicolon
	case "'":
		return ebiten.KeyApostrophe
	case ",":
		return ebiten.KeyComma
	case ".":
		return ebiten.KeyPeriod
	case "/":
		return ebiten.KeySlash
	}
	if len(name) != 1 {
		panic("not reached: unknown key " + name)
	}
	c := name[0]
	if '0' <= c && c <= '9' {
		return ebiten.Key0 + ebiten.Key(c-'0')
	}
	if 'A' <= c && c <= 'Z' {
		return ebiten.KeyA + ebiten.Key(c-'A')
	}
	panic("not reached: unknown key " + name)
}

func drawKey(t *ebiten.Image, name string, x, y, width int) {
	const height = 16
	width--
	img, _ := ebiten.NewImage(width, height, ebiten.FilterDefault)
	p := make([]byte, width*height*4)
	for j := 0; j < height; j++ {
		for i := 0; i < width; i++ {
			x := (i + j*width) * 4
			switch j {
			case 0, height - 1:
				if 3 <= i && i <= width-4 {
					p[x] = 0xff
					p[x+1] = 0xff
					p[x+2] = 0xff
					p[x+3] = 0xff
				}
			case 1, height - 2:
				if i == 2 || i == width-3 {
					p[x] = 0xff
					p[x+1] = 0xff
					p[x+2] = 0xff
					p[x+3] = 0xff
				}
			case 2, height - 3:
				if i == 1 || i == width-2 {
					p[x] = 0xff
					p[x+1] = 0xff
					p[x+2] = 0xff
					p[x+3] = 0xff
				}
			default:
				if i == 0 || i == width-1 {
					p[x] = 0xff
					p[x+1] = 0xff
					p[x+2] = 0xff
					p[x+3] = 0xff
				}
			}
		}
	}
	img.ReplacePixels(p)
	const offset = 4
	text.Draw(img, name, arcadeFont, offset, arcadeFontSize+offset, color.White)

	op := &ebiten.DrawImageOptions{}
	op.GeoM.Translate(float64(x), float64(y))
	t.DrawImage(img, op)
}

func outputKeyboardImage() (map[ebiten.Key]image.Rectangle, error) {
	keyMap := map[ebiten.Key]image.Rectangle{}
	img, _ := ebiten.NewImage(320, 240, ebiten.FilterDefault)
	x, y := 0, 0
	for j, line := range keyboardKeys {
		x = 0
		const height = 18
		for i, keyDisplayName := range line {
			width := 16
			switch j {
			default:
				switch i {
				case 0:
					width = 16 + 8*(j+2)
				case len(line) - 1:
					if j > 0 {
						width = 16 + 8*(j+2)
					}
				}
			case 4:
				switch i {
				case 0:
					width = 16 + 8*(j+2)
				case 1:
					width = 16 * 2
				case 2:
					width = 16 * 5
				case 3:
					width = 16 * 2
				case 4:
					width = 16 + 8*(j+2)
				}
			case 6, 7:
				width = 16 * 3
			}
			if keyDisplayName != "" {
				drawKey(img, keyDisplayName, x, y, width)
				if keyDisplayName != " " {
					key := keyDisplayNameToKey(keyDisplayName)
					keyMap[key] = image.Rect(x, y, x+width, y+height)
				}
			}
			x += width
		}
		y += height
	}

	f, err := ioutil.TempFile("", "ebiten")
	if err != nil {
		return nil, err
	}
	defer f.Close()

	if err := png.Encode(f, img); err != nil {
		return nil, err
	}

	args := []string{
		"-package=keyboard",
		"-input=" + f.Name(),
		"-output=../../resources/images/keyboard/keyboard.go",
		"-var=Keyboard_png",
	}
	if err := exec.Command("file2byteslice", args...).Run(); err != nil {
		return nil, err
	}

	return keyMap, nil
}

const license = `// Copyright 2013 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
`

const keyRectTmpl = `{{.License}}

// Code generated by gen.go using 'go generate'. DO NOT EDIT.

package keyboard

import (
	"image"

	"github.com/hajimehoshi/ebiten"
)

var keyboardKeyRects = map[ebiten.Key]image.Rectangle{}

func init() {
{{range $key, $rect := .KeyRectsMap}}	keyboardKeyRects[ebiten.Key{{$key}}] = image.Rect({{$rect.Min.X}}, {{$rect.Min.Y}}, {{$rect.Max.X}}, {{$rect.Max.Y}})
{{end}}}

func KeyRect(key ebiten.Key) (image.Rectangle, bool) {
	r, ok := keyboardKeyRects[key]
	return r, ok
}`

func outputKeyRectsGo(k map[ebiten.Key]image.Rectangle) error {
	path := "keyrects.go"

	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer f.Close()

	tmpl, err := template.New(path).Parse(keyRectTmpl)
	if err != nil {
		return err
	}
	return tmpl.Execute(f, map[string]interface{}{
		"License":     license,
		"KeyRectsMap": k,
	})
}

var regularTermination = errors.New("regular termination")

func main() {
	var rects map[ebiten.Key]image.Rectangle
	if err := ebiten.Run(func(_ *ebiten.Image) error {
		var err error
		rects, err = outputKeyboardImage()
		if err != nil {
			return err
		}
		return regularTermination
	}, 256, 256, 1, ""); err != regularTermination {
		log.Fatal(err)
	}
	if err := outputKeyRectsGo(rects); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>examples/keyboard/keyboard: Use RunGame<commit_after>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build ignore

package main

import (
	"errors"
	"image"
	"image/color"
	"image/png"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"text/template"

	"github.com/golang/freetype/truetype"
	"golang.org/x/image/font"

	"github.com/hajimehoshi/ebiten"
	"github.com/hajimehoshi/ebiten/text"
)

const (
	arcadeFontSize = 8
)

var (
	arcadeFont font.Face
)

func init() {
	f, err := os.Open(filepath.Join("..", "..", "resources", "fonts", "arcade_n.ttf"))
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	b, err := ioutil.ReadAll(f)
	if err != nil {
		log.Fatal(err)
	}

	tt, err := truetype.Parse(b)
	if err != nil {
		log.Fatal(err)
	}

	const dpi = 72
	arcadeFont = truetype.NewFace(tt, &truetype.Options{
		Size:    arcadeFontSize,
		DPI:     dpi,
		Hinting: font.HintingFull,
	})
}

var keyboardKeys = [][]string{
	{"Esc", "1", "2", "3", "4", "5", "6", "7", "8", "9", "0", "-", "=", "\\", "`", " "},
	{"Tab", "Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P", "[", "]", "BS"},
	{"Ctrl", "A", "S", "D", "F", "G", "H", "J", "K", "L", ";", "'", "Enter"},
	{"Shift", "Z", "X", "C", "V", "B", "N", "M", ",", ".", "/", " "},
	{" ", "Alt", "Space", " ", " "},
	{},
	{"", "Up", ""},
	{"Left", "Down", "Right"},
}

func keyDisplayNameToKey(name string) ebiten.Key {
	switch name {
	case "Esc":
		return ebiten.KeyEscape
	case "Tab":
		return ebiten.KeyTab
	case "Ctrl":
		return ebiten.KeyControl
	case "Shift":
		return ebiten.KeyShift
	case "Alt":
		return ebiten.KeyAlt
	case "Space":
		return ebiten.KeySpace
	case "Up":
		return ebiten.KeyUp
	case "Left":
		return ebiten.KeyLeft
	case "Down":
		return ebiten.KeyDown
	case "Right":
		return ebiten.KeyRight
	case "BS":
		return ebiten.KeyBackspace
	case "Enter":
		return ebiten.KeyEnter
	case "-":
		return ebiten.KeyMinus
	case "=":
		return ebiten.KeyEqual
	case "\\":
		return ebiten.KeyBackslash
	case "`":
		return ebiten.KeyGraveAccent
	case "[":
		return ebiten.KeyLeftBracket
	case "]":
		return ebiten.KeyRightBracket
	case ";":
		return ebiten.KeySemicolon
	case "'":
		return ebiten.KeyApostrophe
	case ",":
		return ebiten.KeyComma
	case ".":
		return ebiten.KeyPeriod
	case "/":
		return ebiten.KeySlash
	}
	if len(name) != 1 {
		panic("not reached: unknown key " + name)
	}
	c := name[0]
	if '0' <= c && c <= '9' {
		return ebiten.Key0 + ebiten.Key(c-'0')
	}
	if 'A' <= c && c <= 'Z' {
		return ebiten.KeyA + ebiten.Key(c-'A')
	}
	panic("not reached: unknown key " + name)
}

func drawKey(t *ebiten.Image, name string, x, y, width int) {
	const height = 16
	width--
	img, _ := ebiten.NewImage(width, height, ebiten.FilterDefault)
	p := make([]byte, width*height*4)
	for j := 0; j < height; j++ {
		for i := 0; i < width; i++ {
			x := (i + j*width) * 4
			switch j {
			case 0, height - 1:
				if 3 <= i && i <= width-4 {
					p[x] = 0xff
					p[x+1] = 0xff
					p[x+2] = 0xff
					p[x+3] = 0xff
				}
			case 1, height - 2:
				if i == 2 || i == width-3 {
					p[x] = 0xff
					p[x+1] = 0xff
					p[x+2] = 0xff
					p[x+3] = 0xff
				}
			case 2, height - 3:
				if i == 1 || i == width-2 {
					p[x] = 0xff
					p[x+1] = 0xff
					p[x+2] = 0xff
					p[x+3] = 0xff
				}
			default:
				if i == 0 || i == width-1 {
					p[x] = 0xff
					p[x+1] = 0xff
					p[x+2] = 0xff
					p[x+3] = 0xff
				}
			}
		}
	}
	img.ReplacePixels(p)
	const offset = 4
	text.Draw(img, name, arcadeFont, offset, arcadeFontSize+offset, color.White)

	op := &ebiten.DrawImageOptions{}
	op.GeoM.Translate(float64(x), float64(y))
	t.DrawImage(img, op)
}

func outputKeyboardImage() (map[ebiten.Key]image.Rectangle, error) {
	keyMap := map[ebiten.Key]image.Rectangle{}
	img, _ := ebiten.NewImage(320, 240, ebiten.FilterDefault)
	x, y := 0, 0
	for j, line := range keyboardKeys {
		x = 0
		const height = 18
		for i, keyDisplayName := range line {
			width := 16
			switch j {
			default:
				switch i {
				case 0:
					width = 16 + 8*(j+2)
				case len(line) - 1:
					if j > 0 {
						width = 16 + 8*(j+2)
					}
				}
			case 4:
				switch i {
				case 0:
					width = 16 + 8*(j+2)
				case 1:
					width = 16 * 2
				case 2:
					width = 16 * 5
				case 3:
					width = 16 * 2
				case 4:
					width = 16 + 8*(j+2)
				}
			case 6, 7:
				width = 16 * 3
			}
			if keyDisplayName != "" {
				drawKey(img, keyDisplayName, x, y, width)
				if keyDisplayName != " " {
					key := keyDisplayNameToKey(keyDisplayName)
					keyMap[key] = image.Rect(x, y, x+width, y+height)
				}
			}
			x += width
		}
		y += height
	}

	f, err := ioutil.TempFile("", "ebiten")
	if err != nil {
		return nil, err
	}
	defer f.Close()

	if err := png.Encode(f, img); err != nil {
		return nil, err
	}

	args := []string{
		"-package=keyboard",
		"-input=" + f.Name(),
		"-output=../../resources/images/keyboard/keyboard.go",
		"-var=Keyboard_png",
	}
	if err := exec.Command("file2byteslice", args...).Run(); err != nil {
		return nil, err
	}

	return keyMap, nil
}

const license = `// Copyright 2013 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
`

const keyRectTmpl = `{{.License}}

// Code generated by gen.go using 'go generate'. DO NOT EDIT.

package keyboard

import (
	"image"

	"github.com/hajimehoshi/ebiten"
)

var keyboardKeyRects = map[ebiten.Key]image.Rectangle{}

func init() {
{{range $key, $rect := .KeyRectsMap}}	keyboardKeyRects[ebiten.Key{{$key}}] = image.Rect({{$rect.Min.X}}, {{$rect.Min.Y}}, {{$rect.Max.X}}, {{$rect.Max.Y}})
{{end}}}

func KeyRect(key ebiten.Key) (image.Rectangle, bool) {
	r, ok := keyboardKeyRects[key]
	return r, ok
}`

func outputKeyRectsGo(k map[ebiten.Key]image.Rectangle) error {
	path := "keyrects.go"

	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer f.Close()

	tmpl, err := template.New(path).Parse(keyRectTmpl)
	if err != nil {
		return err
	}
	return tmpl.Execute(f, map[string]interface{}{
		"License":     license,
		"KeyRectsMap": k,
	})
}

var regularTermination = errors.New("regular termination")

type game struct {
	rects map[ebiten.Key]image.Rectangle
}

func (g *game) Update(_ *ebiten.Image) error {
	var err error
	g.rects, err = outputKeyboardImage()
	if err != nil {
		return err
	}
	return regularTermination
}

func (g *game) Draw(_ *ebiten.Image) {
}

func (g *game) Layout(outw, outh int) (int, int) {
	return 256, 256
}

func main() {
	g := &game{}
	if err := ebiten.RunGame(g); err != regularTermination {
		log.Fatal(err)
	}
	if err := outputKeyRectsGo(g.rects); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"sync"

	"github.com/tikasan/eventory/server/db"
	"github.com/tikasan/eventory/server/define"
	"github.com/tikasan/eventory/server/model"

	"time"

	_ "github.com/go-sql-driver/mysql"
)

type Server struct {
	db *sql.DB
}

func New() *Server {
	return &Server{}
}

func (s *Server) CheckAndInsert() <-chan []model.Event {
	now := time.Now()
	atdn := make([]model.Inserter, define.SERACH_SCOPE)
	connpass := make([]model.Inserter, define.SERACH_SCOPE)
	//doorKeeper := make([]model.Inserter, define.SERACH_SCOPE)
	allInserter := make([]model.Inserter, 0)

	for i := 0; i < define.SERACH_SCOPE; i++ {
		ym := now.AddDate(0, i, 0).Format("200601")
		atdn[i].Url = fmt.Sprintf("https://api.atnd.org/events/?count=100&format=jsonp&callback=&ym=%s", ym)
		atdn[i].Api = define.ATDN

		connpass[i].Url = fmt.Sprintf("https://connpass.com/api/v1/event/?count=100&ym=%s", ym)
		connpass[i].Api = define.CONNPASS

		//doorKeeper[i].Url = fmt.Sprintf("https://api.doorkeeper.jp/events?page=%d", i)
		//doorKeeper[i].Api = define.DOORKEEPER
	}

	allInserter = append(allInserter, atdn...)
	allInserter = append(allInserter, connpass...)
	//allInserter = append(allInserter, doorKeeper...)
	allEvents := make(chan []model.Event, len(allInserter))
	var wg sync.WaitGroup

	go func() {
		for _, a := range allInserter {
			wg.Add(1)
			go func(a model.Inserter) {
				cli := model.NewInserter(a.Url, a.Api)
				events, err := cli.Get()
				if err != nil {
					fmt.Fprint(os.Stderr, err)
				}
				allEvents <- events
				wg.Done()
			}(a)
		}
		wg.Wait()
		close(allEvents)
	}()
	return allEvents
}

func (s *Server) Response() []model.EventJson {
	Event, err := model.EventAllNew(s.db)
	if err != nil {
		fmt.Fprint(os.Stderr, err)
	}
	return Event
}

func (s *Server) Init(dbconf, env string) {
	cs, err := db.NewConfigsFromFile(dbconf)
	if err != nil {
		log.Fatalf("cannot open database configuration. exit. %s", err)
	}
	s.db, err = cs.Open(env)
	if err != nil {
		log.Fatalf("db initialization failed: %s", err)
	}

	_, err = os.Stat("./log")
	if err != nil {
		err := os.Mkdir("./log", 0750)
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			log.Fatalf("log folder initialization failed: %s", err)
		}

	}

	_, err = os.Stat("./log/admin.log")
	if err != nil {
		_, err := os.Create("./log/admin.log")
		if err != nil {
			log.Fatalf("log admin.log initialization failed: %s", err)
		}
	}
}

func (s *Server) Run(port string) {
	http.HandleFunc("/api/pc/events", func(w http.ResponseWriter, r *http.Request) {
		event := s.Response()
		w.Header().Set("Content-Type", "application/json")
		enc := json.NewEncoder(w)
		if err := enc.Encode(event); err != nil {
			http.Error(w, "encoding failed", http.StatusInternalServerError)
			return
		}
	})

	http.HandleFunc("/api/smt/events", func(w http.ResponseWriter, r *http.Request) {
		event := s.Response()
		//for i := range event {
		//	event[i].Desc = formater.RemoveTag(event[i].Desc)
		//}

		w.Header().Set("Content-Type", "application/json")
		enc := json.NewEncoder(w)

		if err := enc.Encode(event); err != nil {
			http.Error(w, "encoding failed", http.StatusInternalServerError)
			return
		}
		fmt.Println("きた")
	})

	http.HandleFunc("/api/events/admin", func(w http.ResponseWriter, r *http.Request) {
		adminLog, err := os.OpenFile("./log/admin.log", os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			return
		}
		defer adminLog.Close()

		now := time.Now()
		logger := log.New(adminLog, "[start]", log.LstdFlags)
		logger.Println(now)
		adminLog.Sync()

		receiver := s.CheckAndInsert()
		for {
			receive, ok := <-receiver
			if ok {
				model.Insert(s.db, receive)
			} else {
				break
			}

		}

		end := time.Now()
		logger = log.New(adminLog, "[end]", log.LstdFlags)
		logger.Println(end)
		adminLog.Sync()
		return
	})
	log.Fatal(http.ListenAndServe(port, nil))
}

func main() {
	var (
		port   = flag.String("port", ":8080", "port to bind")
		dbconf = flag.String("dbconf", "dbconfig.yml", "database configuration file.")
		env    = flag.String("env", "development", "application envirionment (production, development etc.)")
	)

	flag.Parse()
	s := New()

	// イニシャライザが走る
	s.Init(*dbconf, *env)
	s.Run(*port)
}
<commit_msg>[ios][fix]mainパッケージの関数の順番を見やすく直した<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"sync"

	"github.com/tikasan/eventory/server/db"
	"github.com/tikasan/eventory/server/define"
	"github.com/tikasan/eventory/server/model"

	"time"

	_ "github.com/go-sql-driver/mysql"
)

type Server struct {
	db *sql.DB
}

func New() *Server {
	return &Server{}
}

func (s *Server) Init(dbconf, env string) {
	cs, err := db.NewConfigsFromFile(dbconf)
	if err != nil {
		log.Fatalf("cannot open database configuration. exit. %s", err)
	}
	s.db, err = cs.Open(env)
	if err != nil {
		log.Fatalf("db initialization failed: %s", err)
	}

	_, err = os.Stat("./log")
	if err != nil {
		err := os.Mkdir("./log", 0750)
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			log.Fatalf("log folder initialization failed: %s", err)
		}

	}

	_, err = os.Stat("./log/admin.log")
	if err != nil {
		_, err := os.Create("./log/admin.log")
		if err != nil {
			log.Fatalf("log admin.log initialization failed: %s", err)
		}
	}
}

func (s *Server) Run(port string) {
	http.HandleFunc("/api/pc/events", func(w http.ResponseWriter, r *http.Request) {
		event := s.Response()
		w.Header().Set("Content-Type", "application/json")
		enc := json.NewEncoder(w)
		if err := enc.Encode(event); err != nil {
			http.Error(w, "encoding failed", http.StatusInternalServerError)
			return
		}
	})

	http.HandleFunc("/api/smt/events", func(w http.ResponseWriter, r *http.Request) {
		event := s.Response()
		//for i := range event {
		//	event[i].Desc = formater.RemoveTag(event[i].Desc)
		//}

		w.Header().Set("Content-Type", "application/json")
		enc := json.NewEncoder(w)

		if err := enc.Encode(event); err != nil {
			http.Error(w, "encoding failed", http.StatusInternalServerError)
			return
		}
		fmt.Println("きた")
	})

	http.HandleFunc("/api/events/admin", func(w http.ResponseWriter, r *http.Request) {
		adminLog, err := os.OpenFile("./log/admin.log", os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			return
		}
		defer adminLog.Close()

		now := time.Now()
		logger := log.New(adminLog, "[start]", log.LstdFlags)
		logger.Println(now)
		adminLog.Sync()

		receiver := s.CheckAndInsert()
		for {
			receive, ok := <-receiver
			if ok {
				model.Insert(s.db, receive)
			} else {
				break
			}

		}

		end := time.Now()
		logger = log.New(adminLog, "[end]", log.LstdFlags)
		logger.Println(end)
		adminLog.Sync()
		return
	})
	log.Fatal(http.ListenAndServe(port, nil))
}

func (s *Server) CheckAndInsert() <-chan []model.Event {
	now := time.Now()
	atdn := make([]model.Inserter, define.SERACH_SCOPE)
	connpass := make([]model.Inserter, define.SERACH_SCOPE)
	//doorKeeper := make([]model.Inserter, define.SERACH_SCOPE)
	allInserter := make([]model.Inserter, 0)

	for i := 0; i < define.SERACH_SCOPE; i++ {
		ym := now.AddDate(0, i, 0).Format("200601")
		atdn[i].Url = fmt.Sprintf("https://api.atnd.org/events/?count=100&format=jsonp&callback=&ym=%s", ym)
		atdn[i].Api = define.ATDN

		connpass[i].Url = fmt.Sprintf("https://connpass.com/api/v1/event/?count=100&ym=%s", ym)
		connpass[i].Api = define.CONNPASS

		//doorKeeper[i].Url = fmt.Sprintf("https://api.doorkeeper.jp/events?page=%d", i)
		//doorKeeper[i].Api = define.DOORKEEPER
	}

	allInserter = append(allInserter, atdn...)
	allInserter = append(allInserter, connpass...)
	//allInserter = append(allInserter, doorKeeper...)
	allEvents := make(chan []model.Event, len(allInserter))
	var wg sync.WaitGroup

	go func() {
		for _, a := range allInserter {
			wg.Add(1)
			go func(a model.Inserter) {
				cli := model.NewInserter(a.Url, a.Api)
				events, err := cli.Get()
				if err != nil {
					fmt.Fprint(os.Stderr, err)
				}
				allEvents <- events
				wg.Done()
			}(a)
		}
		wg.Wait()
		close(allEvents)
	}()
	return allEvents
}

func (s *Server) Response() []model.EventJson {
	Event, err := model.EventAllNew(s.db)
	if err != nil {
		fmt.Fprint(os.Stderr, err)
	}
	return Event
}

func main() {
	var (
		port   = flag.String("port", ":8080", "port to bind")
		dbconf = flag.String("dbconf", "dbconfig.yml", "database configuration file.")
		env    = flag.String("env", "development", "application envirionment (production, development etc.)")
	)

	flag.Parse()
	s := New()

	// イニシャライザが走る
	s.Init(*dbconf, *env)
	s.Run(*port)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"code.google.com/p/go.net/websocket"
	"github.com/sbinet/liner"
)

var globalGame = NewGame()

func handler(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "."+r.URL.Path)
}

func getClientName(config *websocket.Config) string {
	path := config.Location.Path
	bits := strings.Split(path, "/")
	if len(bits) < 4 {
		return ""
	}
	return bits[3]
}

func mainSocketHandler(ws *websocket.Conn) {
	conn := NewConn(ws)

	msg, err := conn.Recv()
	if err != nil {
		log.Println("Error connecting client, unable to read handshake message: ", err)
		return
	}

	baseName := msg.(*MsgHandshakeInit).DesiredName
	if baseName == "" {
		baseName = "guest"
	}

	name := baseName
	nameNumber := 1
	var client *Client
	for {
		var isNew bool
		client, isNew = globalGame.clientWithID(name)
		if isNew {
			break
		}
		name = fmt.Sprintf("%s-%d", baseName, nameNumber)
		nameNumber++
	}

	conn.Send(&MsgHandshakeReply{
		ServerTime: float64(time.Now().UnixNano()) / 1e6,
		ClientID:   name,
	})

	client.Run(conn)
}

func chunkSocketHandler(ws *websocket.Conn) {
	name := getClientName(ws.Config())

	client, isNew := globalGame.clientWithID(name)
	if isNew {
		log.Println("Warning: Attempt to connect to chunk socket for client '" + name + "' who is not connected on main socket!")
		globalGame.Disconnect(name, "invalid connection")
		return
	}
	client.RunChunks(NewConn(ws))
}

func doProfile() {
	f, err := os.Create("cpuprofile")
	if err != nil {
		log.Fatal(err)
	}
	pprof.StartCPUProfile(f)

	go func() {
		cycles := 4
		for i := 0; i < cycles; i++ {
			log.Print((cycles-i)*30, " seconds left")
			<-time.After(30 * time.Second)
		}
		pprof.StopCPUProfile()
		log.Print("Done! Exiting...")
		os.Exit(1)
	}()
}

func setupPrompt() {
	quit := make(chan bool)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)

	state := liner.NewLiner()
	go promptLoop(quit, state)

	go func() {
		<-c
		fmt.Println()
		quit <- true
	}()

	go func() {
		<-quit
		state.Close()
		os.Exit(0)
	}()
}

func promptLoop(quit chan bool, state *liner.State) {
	for {
		cmd, err := state.Prompt(" >>> ")
		state.AppendHistory(cmd)
		if err != nil {
			fmt.Println()
			log.Println("ERROR:", err)
			quit <- true
			return
		}
		if cmd == "exit" {
			quit <- true
			return
		}
	}
}

func main() {
// 	setupPrompt()

	runtime.GOMAXPROCS(runtime.NumCPU())
	go globalGame.Run()
// 	go doProfile()

	http.HandleFunc("/", handler)
	http.Handle("/sockets/main/", websocket.Handler(mainSocketHandler))
	http.Handle("/sockets/chunk/", websocket.Handler(chunkSocketHandler))

	err := http.ListenAndServe(":8080", nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}
<commit_msg>Add timeout for crash testing.<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"code.google.com/p/go.net/websocket"
	"github.com/sbinet/liner"
)

var globalGame = NewGame()

func handler(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "."+r.URL.Path)
}

func getClientName(config *websocket.Config) string {
	path := config.Location.Path
	bits := strings.Split(path, "/")
	if len(bits) < 4 {
		return ""
	}
	return bits[3]
}

func mainSocketHandler(ws *websocket.Conn) {
	conn := NewConn(ws)
	clientChan := make(chan *Client)
	go func() {
		clientChan <- doHandshake(conn)
	}()

	select {
	case client := <-clientChan:
		client.Run(conn)
	case <-time.After(time.Second):
		fatalExitChan <- "Client connection timeout."
	}
}

func doHandshake(conn *Conn) *Client {
	msg, err := conn.Recv()
	if err != nil {
		log.Println("Error connecting client, unable to read handshake message: ", err)
		return nil
	}

	baseName := msg.(*MsgHandshakeInit).DesiredName
	if baseName == "" {
		baseName = "guest"
	}

	name := baseName
	nameNumber := 1
	var client *Client
	for {
		var isNew bool
		client, isNew = globalGame.clientWithID(name)
		if isNew {
			break
		}
		name = fmt.Sprintf("%s-%d", baseName, nameNumber)
		nameNumber++
	}

	conn.Send(&MsgHandshakeReply{
		ServerTime: float64(time.Now().UnixNano()) / 1e6,
		ClientID:   name,
	})
	return client
}

func chunkSocketHandler(ws *websocket.Conn) {
	name := getClientName(ws.Config())

	client, isNew := globalGame.clientWithID(name)
	if isNew {
		log.Println("Warning: Attempt to connect to chunk socket for client '" + name + "' who is not connected on main socket!")
		globalGame.Disconnect(name, "invalid connection")
		return
	}
	client.RunChunks(NewConn(ws))
}

func doProfile() {
	f, err := os.Create("cpuprofile")
	if err != nil {
		log.Fatal(err)
	}
	pprof.StartCPUProfile(f)

	go func() {
		cycles := 4
		for i := 0; i < cycles; i++ {
			log.Print((cycles-i)*30, " seconds left")
			<-time.After(30 * time.Second)
		}
		pprof.StopCPUProfile()
		log.Print("Done! Exiting...")
		os.Exit(1)
	}()
}

func setupPrompt() {
	quit := make(chan bool)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)

	state := liner.NewLiner()
	go promptLoop(quit, state)

	go func() {
		<-c
		fmt.Println()
		quit <- true
	}()

	go func() {
		<-quit
		state.Close()
		os.Exit(0)
	}()
}

func promptLoop(quit chan bool, state *liner.State) {
	for {
		cmd, err := state.Prompt(" >>> ")
		state.AppendHistory(cmd)
		if err != nil {
			fmt.Println()
			log.Println("ERROR:", err)
			quit <- true
			return
		}
		if cmd == "exit" {
			quit <- true
			return
		}
	}
}

func fatalExit() {
	reason := <- fatalExitChan
	panic(reason)
}

var fatalExitChan chan string

func main() {
	fatalExitChan = make(chan string)
	go fatalExit()
// 	setupPrompt()

	runtime.GOMAXPROCS(runtime.NumCPU())
	go globalGame.Run()
// 	go doProfile()

	http.HandleFunc("/", handler)
	http.Handle("/sockets/main/", websocket.Handler(mainSocketHandler))
	http.Handle("/sockets/chunk/", websocket.Handler(chunkSocketHandler))

	err := http.ListenAndServe(":8080", nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}
<|endoftext|>