<commit_before>package adapter

import (
	"reflect"
	"text/template"
)

const (
	PlanStandalone       = "standalone"
	PlanShardedSet       = "sharded_set"
	PlanSingleReplicaSet = "single_replica_set"
)

var plans = map[string]*template.Template{}

func init() {
	funcs := template.FuncMap{
		"last": func(a interface{}, x int) bool {
			return reflect.ValueOf(a).Len()-1 == x
		},
	}

	var err error
	for k, s := range plansRaw {
		plans[k], err = template.New(string(k)).Funcs(funcs).Parse(s)
		if err != nil {
			panic(err)
		}
	}
}

var plansRaw = map[string]string{
	PlanStandalone: `{
    "options": {
        "downloadBase": "/var/lib/mongodb-mms-automation",
        "downloadBaseWindows": "C:\\mongodb-mms-automation"
    },
    "mongoDbVersions": [{
        "builds": [
                {
                    "bits": 64,
                    "flavor": "",
                    "gitVersion": "4249c1d2b5999ebbf1fdf3bc0e0e3b3ff5c0aaf2",
                    "maxOsVersion": "",
                    "minOsVersion": "",
                    "modules": [],
                    "platform": "osx",
                    "url": "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.2.7.tgz",
                    "win2008plus": false,
                    "winVCRedistDll": "",
                    "winVCRedistOptions": [],
                    "winVCRedistUrl": "",
                    "winVCRedistVersion": ""
                }
            ],
        "name": "{{.Version}}"
    }],
    "backupVersions": [{
        "hostname": "{{index .Nodes 0}}",
        "logPath": "/var/vcap/sys/log/mongod_node/backup-agent.log",
        "logRotate": {
            "sizeThresholdMB": 1000,
            "timeThresholdHrs": 24
        }
    }],

    "monitoringVersions": [{
        "hostname": "{{index .Nodes 0}}",
        "logPath": "/var/vcap/sys/log/mongod_node/monitoring-agent.log",
        "logRotate": {
            "sizeThresholdMB": 1000,
            "timeThresholdHrs": 24
        }
    }],
    "processes": [{
        "args2_6": {
            "net": {
                "port": 28000
            },
            "storage": {
                "dbPath": "/var/vcap/store/mongodb-data"
            },
            "systemLog": {
                "destination": "file",
                "path": "/var/vcap/sys/log/mongod_node/mongodb.log"
            }
        },
        "hostname": "{{index .Nodes 0}}",
        "logRotate": {
            "sizeThresholdMB": 1000,
            "timeThresholdHrs": 24
        },
        "name": "{{index .Nodes 0}}",
        "processType": "mongod",
        "version": "{{.Version}}",
        "authSchemaVersion": 5
    }],
    "replicaSets": [],
    "roles": [],
    "sharding": [],

    "auth": {
        "autoUser": "mms-automation",
        "autoPwd": "{{.Password}}",
        "deploymentAuthMechanisms": [
            "SCRAM-SHA-1"
        ],
        "key": "{{.Key}}",
        "keyfile": "/var/vcap/jobs/mongod_node/config/mongo_om.key",
        "disabled": false,
        "usersDeleted": [],
        "usersWanted": [
            {
                "db": "admin",
                "roles": [
                    {
                        "db": "admin",
                        "role": "clusterMonitor"
                    }
                ],
                "user": "mms-monitoring-agent",
                "initPwd": "{{.Password}}"
            },
            {
                "db": "admin",
                "roles": [
                    {
                        "db": "admin",
                        "role": "clusterAdmin"
                    },
                    {
                        "db": "admin",
                        "role": "readAnyDatabase"
                    },
                    {
                        "db": "admin",
                        "role": "userAdminAnyDatabase"
                    },
                    {
                        "db": "local",
                        "role": "readWrite"
                    },
                    {
                        "db": "admin",
                        "role": "readWrite"
                    }
                ],
                "user": "mms-backup-agent",
                "initPwd": "{{.Password}}"
            },
            {
               "db": "admin" ,
               "user": "admin" ,
               "roles": [
                 {
                     "db": "admin",
                     "role": "clusterAdmin"
                 },
                 {
                     "db": "admin",
                     "role": "readAnyDatabase"
                 },
                 {
                     "db": "admin",
                     "role": "userAdminAnyDatabase"
                 },
                 {
                     "db": "local",
                     "role": "readWrite"
                 },
                 {
                     "db": "admin",
                     "role": "readWrite"
                 }
               ],
               "initPwd": "{{.AdminPassword}}"
            }
        ],
        "autoAuthMechanism": "SCRAM-SHA-1"
    }
}`,

	PlanShardedSet: `{
    "options": {
        "downloadBase": "/var/lib/mongodb-mms-automation",
        "downloadBaseWindows": "C:\\mongodb-mms-automation"
    },
    "mongoDbVersions": [{
        "name": "{{.Version}}"
    }],
    "backupVersions": [
    ],

    "monitoringVersions": [
    {
        "hostname": "{{index .Cluster.Routers 0}}",
        "logPath": "/var/vcap/sys/log/mongod_node/monitoring-agent.log",
        "logRotate": {
            "sizeThresholdMB": 1000,
            "timeThresholdHrs": 24
        }
    }
    ],
    "processes": [
      {{range $i, $node := .Cluster.Routers}}{
          "args2_6": {
              "net": {
                  "port": 28000
              },
              "systemLog": {
                  "destination": "file",
                  "path": "/var/vcap/sys/log/mongod_node/mongodb.log"
              }
          },
          "name": "{{$node}}",
          "hostname": "{{$node}}",
          "logRotate": {
              "sizeThresholdMB": 1000,
              "timeThresholdHrs": 24
          },
          "version": "{{$.Version}}",
          "authSchemaVersion": 5,
          "processType": "mongos",
          "cluster": "{{$.ID}}_cluster"
      },{{end}}

      {{range $i, $node := .Cluster.ConfigServers}}{
          "args2_6": {
              "net": {
                  "port": 28000
              },
              "replication": {
                  "replSetName": "{{$.ID}}_config"
              },
              "sharding": {
                "clusterRole": "configsvr"
              },
              "storage": {
                  "dbPath": "/var/vcap/store/mongodb-data"
              },
              "systemLog": {
                  "destination": "file",
                  "path": "/var/vcap/sys/log/mongod_node/mongodb.log"
              }
          },
          "name": "{{$node}}",
          "hostname": "{{$node}}",
          "logRotate": {
              "sizeThresholdMB": 1000,
              "timeThresholdHrs": 24
          },
          "version": "{{$.Version}}",
          "authSchemaVersion": 5,
          "processType": "mongod"
      }{{if last $.Cluster.ConfigServers $i}}{{else}},{{end}}{{end}}

      {{range $ii, $shard := .Cluster.Shards}}
          {{range $i, $node := $shard}},{
              "args2_6": {
                  "net": {
                      "port": 28000
                  },
                  "replication": {
                      "replSetName": "{{$.ID}}_shard_{{$ii}}"
                  },
                  "storage": {
                       "dbPath": "/var/vcap/store/mongodb-data"
                  },
                  "systemLog": {
                      "destination": "file",
                      "path": "/var/vcap/sys/log/mongod_node/mongodb.log"
                  }
              },
              "name": "{{$node}}",
              "hostname": "{{$node}}",
              "logRotate": {
                  "sizeThresholdMB": 1000,
                  "timeThresholdHrs": 24
              },
              "version": "{{$.Version}}",
              "authSchemaVersion": 5,
              "processType": "mongod"
          }{{end}}
      {{end}}
    ],

    "replicaSets": [{
        "_id": "{{$.ID}}_config",
        "members": [
            {{range $i, $node := .Cluster.ConfigServers}}{{if $i}},{{end}}{
                "_id": {{$i}},
                "arbiterOnly": false,
                "hidden": false,
                "host": "{{$node}}",
                "priority": 1,
                "slaveDelay": 0,
                "votes": 1
            }{{end}}
        ]
    }
    {{range $i, $shard := .Cluster.Shards}},{
        "_id": "shard_{{$.ID}}_{{$i}}",
        "members": [{{range $i, $node := $shard}}
            {{if $i}},{{end}}{
                "_id": {{$i}},
                "arbiterOnly": false,
                "hidden": false,
                "host": "{{$node}}",
                "priority": 1,
                "slaveDelay": 0,
                "votes": 1
            }
            {{end}}
        ]
    }{{end}}],

    "sharding": [{
        "shards": [
             {{range $i, $shard := .Cluster.Shards}}{{if $i}},{{end}}{
                 "tags": [],
                 "_id": "{{$.ID}}_shard_{{$i}}",
                 "rs": "{{$.ID}}_shard_{{$i}}"
             }{{end}}
        ],
        "name": "{{.ID}}_cluster",
        "configServer": [],
        "configServerReplica": "{{.ID}}_config",
        "collections": []
    }],

    "auth":{
       "disabled":false,
       "autoPwd": "{{.Password}}",
       "autoUser":"mms-automation",
       "deploymentAuthMechanisms": [
           "MONGODB-CR"
       ],
       "key":"{{.Key}}",
       "keyfile":"/var/vcap/jobs/mongod_node/config/mongo_om.key",
       "usersWanted":[
          {
             "db":"admin",
             "initPwd":"{{.Password}}",
             "roles":[
                {
                   "db":"admin",
                   "role":"clusterMonitor"
                }
             ],
             "user":"mms-monitoring-agent"
          },
          {
             "db":"admin",
             "initPwd":"{{.Password}}",
             "roles":[
                {
                   "db":"admin",
                   "role":"clusterAdmin"
                },
                {
                   "db":"admin",
                   "role":"readAnyDatabase"
                },
                {
                   "db":"admin",
                   "role":"userAdminAnyDatabase"
                },
                {
                   "db":"local",
                   "role":"readWrite"
                },
                {
                   "db":"admin",
                   "role":"readWrite"
                }
             ],
             "user":"mms-backup-agent"
          }
       ],
       "usersDeleted":[],
       "autoAuthMechanism": "MONGODB-CR"
    }
}`,

	PlanSingleReplicaSet: `{
    "options": {
        "downloadBase": "/var/lib/mongodb-mms-automation",
        "downloadBaseWindows": "C:\\mongodb-mms-automation"
    },
    "mongoDbVersions": [{
        "builds": [
                {
                    "bits": 64,
                    "flavor": "",
                    "gitVersion": "4249c1d2b5999ebbf1fdf3bc0e0e3b3ff5c0aaf2",
                    "maxOsVersion": "",
                    "minOsVersion": "",
                    "modules": [],
                    "platform": "osx",
                    "url": "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.2.7.tgz",
                    "win2008plus": false,
                    "winVCRedistDll": "",
                    "winVCRedistOptions": [],
                    "winVCRedistUrl": "",
                    "winVCRedistVersion": ""
                }
            ],
        "name": "{{.Version}}"
    }],
    "backupVersions": [{
        "hostname": "{{index .Nodes 0}}",
        "logPath": "/var/vcap/sys/log/mongod_node/backup-agent.log",
        "logRotate": {
            "sizeThresholdMB": 1000,
            "timeThresholdHrs": 24
        }
    }],

    "monitoringVersions": [{
        "hostname": "{{index .Nodes 0}}",
        "logPath": "/var/vcap/sys/log/mongod_node/monitoring-agent.log",
        "logRotate": {
            "sizeThresholdMB": 1000,
            "timeThresholdHrs": 24
        }
    }],
    "processes": [{{range $i, $node := .Nodes}}
      {{if $i}},{{end}}{
        "args2_6": {
            "net": {
                "port": 28000
            },
            "replication": {
                "replSetName": "pcf_repl"
            },
            "storage": {
                "dbPath": "/var/vcap/store/mongodb-data"
            },
            "systemLog": {
                "destination": "file",
                "path": "/var/vcap/sys/log/mongod_node/mongodb.log"
            }
        },
        "hostname": "{{$node}}",
        "logRotate": {
            "sizeThresholdMB": 1000,
            "timeThresholdHrs": 24
        },
        "name": "{{$node}}",
        "processType": "mongod",
        "version": "{{$.Version}}",
        "authSchemaVersion": 5
    }
    {{end}}
  ],
    "replicaSets": [{
        "_id": "pcf_repl",
        "members": [
          {{range $i, $node := .Nodes}}
          {{if $i}},{{end}}{
            "_id": {{$i}},
            "host": "{{$node}}"{{if last $.Nodes $i}},
            "arbiterOnly": true,
            "priority": 0
            {{end}}
          }
          {{end}}
        ]
    }],
    "roles": [],
    "sharding": [],

    "auth": {
        "autoUser": "mms-automation",
        "autoPwd": "{{.Password}}",
        "deploymentAuthMechanisms": [
            "SCRAM-SHA-1"
        ],
        "key": "{{.Key}}",
        "keyfile": "/var/vcap/jobs/mongod_node/config/mongo_om.key",
        "disabled": false,
        "usersDeleted": [],
        "usersWanted": [
            {
                "db": "admin",
                "roles": [
                    {
                        "db": "admin",
                        "role": "clusterMonitor"
                    }
                ],
                "user": "mms-monitoring-agent",
                "initPwd": "{{.Password}}"
            },
            {
                "db": "admin",
                "roles": [
                    {
                        "db": "admin",
                        "role": "clusterAdmin"
                    },
                    {
                        "db": "admin",
                        "role": "readAnyDatabase"
                    },
                    {
                        "db": "admin",
                        "role": "userAdminAnyDatabase"
                    },
                    {
                        "db": "local",
                        "role": "readWrite"
                    },
                    {
                        "db": "admin",
                        "role": "readWrite"
                    }
                ],
                "user": "mms-backup-agent",
                "initPwd": "{{.Password}}"
            },
            {
               "db": "admin" ,
               "user": "admin" ,
               "roles": [
                 {
                     "db": "admin",
                     "role": "clusterAdmin"
                 },
                 {
                     "db": "admin",
                     "role": "readAnyDatabase"
                 },
                 {
                     "db": "admin",
                     "role": "userAdminAnyDatabase"
                 },
                 {
                     "db": "local",
                     "role": "readWrite"
                 },
                 {
                     "db": "admin",
                     "role": "readWrite"
                 }
               ],
               "initPwd": "{{.AdminPassword}}"
            }
        ],
        "autoAuthMechanism": "SCRAM-SHA-1"
    }
}`,
}
<commit_msg>Fix shard id<commit_after>package adapter

import (
	"reflect"
	"text/template"
)

const (
	PlanStandalone       = "standalone"
	PlanShardedSet       = "sharded_set"
	PlanSingleReplicaSet = "single_replica_set"
)

var plans = map[string]*template.Template{}

func init() {
	funcs := template.FuncMap{
		"last": func(a interface{}, x int) bool {
			return reflect.ValueOf(a).Len()-1 == x
		},
	}

	var err error
	for k, s := range plansRaw {
		plans[k], err = template.New(string(k)).Funcs(funcs).Parse(s)
		if err != nil {
			panic(err)
		}
	}
}

var plansRaw = map[string]string{
	PlanStandalone: `{
    "options": {
        "downloadBase": "/var/lib/mongodb-mms-automation",
        "downloadBaseWindows": "C:\\mongodb-mms-automation"
    },
    "mongoDbVersions": [{
        "builds": [
                {
                    "bits": 64,
                    "flavor": "",
                    "gitVersion": "4249c1d2b5999ebbf1fdf3bc0e0e3b3ff5c0aaf2",
                    "maxOsVersion": "",
                    "minOsVersion": "",
                    "modules": [],
                    "platform": "osx",
                    "url": "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.2.7.tgz",
                    "win2008plus": false,
                    "winVCRedistDll": "",
                    "winVCRedistOptions": [],
                    "winVCRedistUrl": "",
                    "winVCRedistVersion": ""
                }
            ],
        "name": "{{.Version}}"
    }],
    "backupVersions": [{
        "hostname": "{{index .Nodes 0}}",
        "logPath": "/var/vcap/sys/log/mongod_node/backup-agent.log",
        "logRotate": {
            "sizeThresholdMB": 1000,
            "timeThresholdHrs": 24
        }
    }],

    "monitoringVersions": [{
        "hostname": "{{index .Nodes 0}}",
        "logPath": "/var/vcap/sys/log/mongod_node/monitoring-agent.log",
        "logRotate": {
            "sizeThresholdMB": 1000,
            "timeThresholdHrs": 24
        }
    }],
    "processes": [{
        "args2_6": {
            "net": {
                "port": 28000
            },
            "storage": {
                "dbPath": "/var/vcap/store/mongodb-data"
            },
            "systemLog": {
                "destination": "file",
                "path": "/var/vcap/sys/log/mongod_node/mongodb.log"
            }
        },
        "hostname": "{{index .Nodes 0}}",
        "logRotate": {
            "sizeThresholdMB": 1000,
            "timeThresholdHrs": 24
        },
        "name": "{{index .Nodes 0}}",
        "processType": "mongod",
        "version": "{{.Version}}",
        "authSchemaVersion": 5
    }],
    "replicaSets": [],
    "roles": [],
    "sharding": [],

    "auth": {
        "autoUser": "mms-automation",
        "autoPwd": "{{.Password}}",
        "deploymentAuthMechanisms": [
            "SCRAM-SHA-1"
        ],
        "key": "{{.Key}}",
        "keyfile": "/var/vcap/jobs/mongod_node/config/mongo_om.key",
        "disabled": false,
        "usersDeleted": [],
        "usersWanted": [
            {
                "db": "admin",
                "roles": [
                    {
                        "db": "admin",
                        "role": "clusterMonitor"
                    }
                ],
                "user": "mms-monitoring-agent",
                "initPwd": "{{.Password}}"
            },
            {
                "db": "admin",
                "roles": [
                    {
                        "db": "admin",
                        "role": "clusterAdmin"
                    },
                    {
                        "db": "admin",
                        "role": "readAnyDatabase"
                    },
                    {
                        "db": "admin",
                        "role": "userAdminAnyDatabase"
                    },
                    {
                        "db": "local",
                        "role": "readWrite"
                    },
                    {
                        "db": "admin",
                        "role": "readWrite"
                    }
                ],
                "user": "mms-backup-agent",
                "initPwd": "{{.Password}}"
            },
            {
               "db": "admin" ,
               "user": "admin" ,
               "roles": [
                 {
                     "db": "admin",
                     "role": "clusterAdmin"
                 },
                 {
                     "db": "admin",
                     "role": "readAnyDatabase"
                 },
                 {
                     "db": "admin",
                     "role": "userAdminAnyDatabase"
                 },
                 {
                     "db": "local",
                     "role": "readWrite"
                 },
                 {
                     "db": "admin",
                     "role": "readWrite"
                 }
               ],
               "initPwd": "{{.AdminPassword}}"
            }
        ],
        "autoAuthMechanism": "SCRAM-SHA-1"
    }
}`,

	PlanShardedSet: `{
    "options": {
        "downloadBase": "/var/lib/mongodb-mms-automation",
        "downloadBaseWindows": "C:\\mongodb-mms-automation"
    },
    "mongoDbVersions": [{
        "name": "{{.Version}}"
    }],
    "backupVersions": [
    ],

    "monitoringVersions": [
    {
        "hostname": "{{index .Cluster.Routers 0}}",
        "logPath": "/var/vcap/sys/log/mongod_node/monitoring-agent.log",
        "logRotate": {
            "sizeThresholdMB": 1000,
            "timeThresholdHrs": 24
        }
    }
    ],
    "processes": [
      {{range $i, $node := .Cluster.Routers}}{
          "args2_6": {
              "net": {
                  "port": 28000
              },
              "systemLog": {
                  "destination": "file",
                  "path": "/var/vcap/sys/log/mongod_node/mongodb.log"
              }
          },
          "name": "{{$node}}",
          "hostname": "{{$node}}",
          "logRotate": {
              "sizeThresholdMB": 1000,
              "timeThresholdHrs": 24
          },
          "version": "{{$.Version}}",
          "authSchemaVersion": 5,
          "processType": "mongos",
          "cluster": "{{$.ID}}_cluster"
      },{{end}}

      {{range $i, $node := .Cluster.ConfigServers}}{
          "args2_6": {
              "net": {
                  "port": 28000
              },
              "replication": {
                  "replSetName": "{{$.ID}}_config"
              },
              "sharding": {
                "clusterRole": "configsvr"
              },
              "storage": {
                  "dbPath": "/var/vcap/store/mongodb-data"
              },
              "systemLog": {
                  "destination": "file",
                  "path": "/var/vcap/sys/log/mongod_node/mongodb.log"
              }
          },
          "name": "{{$node}}",
          "hostname": "{{$node}}",
          "logRotate": {
              "sizeThresholdMB": 1000,
              "timeThresholdHrs": 24
          },
          "version": "{{$.Version}}",
          "authSchemaVersion": 5,
          "processType": "mongod"
      }{{if last $.Cluster.ConfigServers $i}}{{else}},{{end}}{{end}}

      {{range $ii, $shard := .Cluster.Shards}}
          {{range $i, $node := $shard}},{
              "args2_6": {
                  "net": {
                      "port": 28000
                  },
                  "replication": {
                      "replSetName": "{{$.ID}}_shard_{{$ii}}"
                  },
                  "storage": {
                       "dbPath": "/var/vcap/store/mongodb-data"
                  },
                  "systemLog": {
                      "destination": "file",
                      "path": "/var/vcap/sys/log/mongod_node/mongodb.log"
                  }
              },
              "name": "{{$node}}",
              "hostname": "{{$node}}",
              "logRotate": {
                  "sizeThresholdMB": 1000,
                  "timeThresholdHrs": 24
              },
              "version": "{{$.Version}}",
              "authSchemaVersion": 5,
              "processType": "mongod"
          }{{end}}
      {{end}}
    ],

    "replicaSets": [{
        "_id": "{{$.ID}}_config",
        "members": [
            {{range $i, $node := .Cluster.ConfigServers}}{{if $i}},{{end}}{
                "_id": {{$i}},
                "arbiterOnly": false,
                "hidden": false,
                "host": "{{$node}}",
                "priority": 1,
                "slaveDelay": 0,
                "votes": 1
            }{{end}}
        ]
    }
    {{range $i, $shard := .Cluster.Shards}},{
        "_id": "{{$.ID}}_shard_{{$i}}",
        "members": [{{range $i, $node := $shard}}
            {{if $i}},{{end}}{
                "_id": {{$i}},
                "arbiterOnly": false,
                "hidden": false,
                "host": "{{$node}}",
                "priority": 1,
                "slaveDelay": 0,
                "votes": 1
            }
            {{end}}
        ]
    }{{end}}],

    "sharding": [{
        "shards": [
             {{range $i, $shard := .Cluster.Shards}}{{if $i}},{{end}}{
                 "tags": [],
                 "_id": "{{$.ID}}_shard_{{$i}}",
                 "rs": "{{$.ID}}_shard_{{$i}}"
             }{{end}}
        ],
        "name": "{{.ID}}_cluster",
        "configServer": [],
        "configServerReplica": "{{.ID}}_config",
        "collections": []
    }],

    "auth":{
       "disabled":false,
       "autoPwd": "{{.Password}}",
       "autoUser":"mms-automation",
       "deploymentAuthMechanisms": [
           "MONGODB-CR"
       ],
       "key":"{{.Key}}",
       "keyfile":"/var/vcap/jobs/mongod_node/config/mongo_om.key",
       "usersWanted":[
          {
             "db":"admin",
             "initPwd":"{{.Password}}",
             "roles":[
                {
                   "db":"admin",
                   "role":"clusterMonitor"
                }
             ],
             "user":"mms-monitoring-agent"
          },
          {
             "db":"admin",
             "initPwd":"{{.Password}}",
             "roles":[
                {
                   "db":"admin",
                   "role":"clusterAdmin"
                },
                {
                   "db":"admin",
                   "role":"readAnyDatabase"
                },
                {
                   "db":"admin",
                   "role":"userAdminAnyDatabase"
                },
                {
                   "db":"local",
                   "role":"readWrite"
                },
                {
                   "db":"admin",
                   "role":"readWrite"
                }
             ],
             "user":"mms-backup-agent"
          }
       ],
       "usersDeleted":[],
       "autoAuthMechanism": "MONGODB-CR"
    }
}`,

	PlanSingleReplicaSet: `{
    "options": {
        "downloadBase": "/var/lib/mongodb-mms-automation",
        "downloadBaseWindows": "C:\\mongodb-mms-automation"
    },
    "mongoDbVersions": [{
        "builds": [
                {
                    "bits": 64,
                    "flavor": "",
                    "gitVersion": "4249c1d2b5999ebbf1fdf3bc0e0e3b3ff5c0aaf2",
                    "maxOsVersion": "",
                    "minOsVersion": "",
                    "modules": [],
                    "platform": "osx",
                    "url": "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.2.7.tgz",
                    "win2008plus": false,
                    "winVCRedistDll": "",
                    "winVCRedistOptions": [],
                    "winVCRedistUrl": "",
                    "winVCRedistVersion": ""
                }
            ],
        "name": "{{.Version}}"
    }],
    "backupVersions": [{
        "hostname": "{{index .Nodes 0}}",
        "logPath": "/var/vcap/sys/log/mongod_node/backup-agent.log",
        "logRotate": {
            "sizeThresholdMB": 1000,
            "timeThresholdHrs": 24
        }
    }],

    "monitoringVersions": [{
        "hostname": "{{index .Nodes 0}}",
        "logPath": "/var/vcap/sys/log/mongod_node/monitoring-agent.log",
        "logRotate": {
            "sizeThresholdMB": 1000,
            "timeThresholdHrs": 24
        }
    }],
    "processes": [{{range $i, $node := .Nodes}}
      {{if $i}},{{end}}{
        "args2_6": {
            "net": {
                "port": 28000
            },
            "replication": {
                "replSetName": "pcf_repl"
            },
            "storage": {
                "dbPath": "/var/vcap/store/mongodb-data"
            },
            "systemLog": {
                "destination": "file",
                "path": "/var/vcap/sys/log/mongod_node/mongodb.log"
            }
        },
        "hostname": "{{$node}}",
        "logRotate": {
            "sizeThresholdMB": 1000,
            "timeThresholdHrs": 24
        },
        "name": "{{$node}}",
        "processType": "mongod",
        "version": "{{$.Version}}",
        "authSchemaVersion": 5
    }
    {{end}}
  ],
    "replicaSets": [{
        "_id": "pcf_repl",
        "members": [
          {{range $i, $node := .Nodes}}
          {{if $i}},{{end}}{
            "_id": {{$i}},
            "host": "{{$node}}"{{if last $.Nodes $i}},
            "arbiterOnly": true,
            "priority": 0
            {{end}}
          }
          {{end}}
        ]
    }],
    "roles": [],
    "sharding": [],

    "auth": {
        "autoUser": "mms-automation",
        "autoPwd": "{{.Password}}",
        "deploymentAuthMechanisms": [
            "SCRAM-SHA-1"
        ],
        "key": "{{.Key}}",
        "keyfile": "/var/vcap/jobs/mongod_node/config/mongo_om.key",
        "disabled": false,
        "usersDeleted": [],
        "usersWanted": [
            {
                "db": "admin",
                "roles": [
                    {
                        "db": "admin",
                        "role": "clusterMonitor"
                    }
                ],
                "user": "mms-monitoring-agent",
                "initPwd": "{{.Password}}"
            },
            {
                "db": "admin",
                "roles": [
                    {
                        "db": "admin",
                        "role": "clusterAdmin"
                    },
                    {
                        "db": "admin",
                        "role": "readAnyDatabase"
                    },
                    {
                        "db": "admin",
                        "role": "userAdminAnyDatabase"
                    },
                    {
                        "db": "local",
                        "role": "readWrite"
                    },
                    {
                        "db": "admin",
                        "role": "readWrite"
                    }
                ],
                "user": "mms-backup-agent",
                "initPwd": "{{.Password}}"
            },
            {
               "db": "admin" ,
               "user": "admin" ,
               "roles": [
                 {
                     "db": "admin",
                     "role": "clusterAdmin"
                 },
                 {
                     "db": "admin",
                     "role": "readAnyDatabase"
                 },
                 {
                     "db": "admin",
                     "role": "userAdminAnyDatabase"
                 },
                 {
                     "db": "local",
                     "role": "readWrite"
                 },
                 {
                     "db": "admin",
                     "role": "readWrite"
                 }
               ],
               "initPwd": "{{.AdminPassword}}"
            }
        ],
        "autoAuthMechanism": "SCRAM-SHA-1"
    }
}`,
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"strconv"
	"sync"
)

func main() {
	var (
		flagDumpRequest  = flag.Bool("d", false, "dump full request to stdout")
		flagLogRequest   = flag.Bool("l", false, "dump request path to stdout")
		flagCountRequest = flag.Bool("c", false, "count requests")
		flagPort         = flag.Uint("p", 0, "port to listen to")
	)
	flag.Parse()
	ln, err := net.Listen("tcp", net.JoinHostPort("0.0.0.0", strconv.Itoa(int(*flagPort))))
	if err != nil {
		log.Fatalf("Could not listen to the port: %v", err)
	}
	log.Printf("Running on %v\n", ln.Addr())

	var (
		mu      sync.Mutex
		counter int64
	)
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if *flagLogRequest {
			log.Printf("%v\n", r.URL)
		}
		if *flagDumpRequest {
			b, _ := httputil.DumpRequest(r, true)
			log.Println(string(b))
		}

		if *flagCountRequest {
			mu.Lock()
			counter++
			mu.Unlock()
			log.Printf("counter: %d\n", counter)
		}
	})
	log.Fatal(http.Serve(ln, nil))
}
<commit_msg>test-server: use custom logger<commit_after>package main

import (
	"flag"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"strconv"
	"sync"
)

func main() {
	var (
		flagDumpRequest  = flag.Bool("d", false, "dump full request to stdout")
		flagLogRequest   = flag.Bool("l", false, "dump request path to stdout")
		flagCountRequest = flag.Bool("c", false, "count requests")
		flagPort         = flag.Uint("p", 0, "port to listen to")
	)
	flag.Parse()

	logger := log.New(os.Stdout, "", log.LstdFlags)

	ln, err := net.Listen("tcp", net.JoinHostPort("0.0.0.0", strconv.Itoa(int(*flagPort))))
	if err != nil {
		logger.Fatalf("Could not listen to the port: %v", err)
	}
	logger.Printf("Running on %v\n", ln.Addr())

	var (
		mu      sync.Mutex
		counter int64
	)
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if *flagLogRequest {
			logger.Printf("%v\n", r.URL)
		}
		if *flagDumpRequest {
			b, _ := httputil.DumpRequest(r, true)
			logger.Println(string(b))
		}

		if *flagCountRequest {
			mu.Lock()
			counter++
			mu.Unlock()
			logger.Printf("counter: %d\n", counter)
		}
	})
	logger.Fatal(http.Serve(ln, nil))
}
<|endoftext|>


<commit_before>package decoders

import (
	"errors"
	"fmt"
	"log"
	"math"
	"regexp"
	"strconv"

	"github.com/seadsystem/Backend/DB/landingzone/constants"
)

type SeadPacket struct {
	Type      byte
	Location  byte
	Timestamp float64
	Period    float64
	Count     int
	Data      float64
	Serial    int
}

var headerRegex *regexp.Regexp
var InvalidHeader = errors.New("Invalid header.")
var InvalidPacket = errors.New("Invalid packet.")
var InvalidTime = errors.New("Invalid time.")

func init() {
	var err error
	headerRegex, err = regexp.Compile(constants.HEADER_REGEX)
	if err != nil {
		log.Panic("Regex compile error:", err)
	}
}

func DecodeHeader(packet []byte) (serial int, err error) {
	serialStrings := headerRegex.FindSubmatch(packet)

	if serialStrings == nil {
		err = InvalidHeader
		return
	}
	log.Printf("Found %d serials.\n", len(serialStrings))
	if len(serialStrings) != 1 {
		err = InvalidHeader
		return
	}
	log.Printf("Header serial string: %s\n", string(serialStrings[0]))

	serial, err = strconv.Atoi(string(serialStrings[0]))
	return
}

// Extract data sent from sensor from request
func DecodePacket(buffer []byte) (packet SeadPacket, err error) {
	for i := 0; i < len(buffer); {
		datatype := buffer[i]
		i++

		// Switch on the type of data sent in the packet
		switch {
		case datatype == 'T':
			// Type
			packet.Type = buffer[i]
			i++
		case datatype == 'l':
			// Location
			packet.Location = buffer[i]
			i++
		case datatype == 't':
			// Timestamp
			packet.Timestamp, err = asciiTimeToDouble(buffer[i : i+14])
			i += 14
		case datatype == 'P':
			// Period separator
			packet.Period, err = asciiTimeToDouble(buffer[i : i+14])
			i += 14
		case datatype == 'C':
			// Count
			packet.Count, err = strconv.Atoi(string(buffer[i : i+2]))
			i += 2
		case datatype == 'D':
			// Data
			// if count isn't set, return error
			// TODO finish parsing data
		case datatype == 'S':
			// Serial
			packet.Serial, err = strconv.Atoi(string(buffer[i : i+6]))
			i += 6
		case datatype == 'X':
			return
		default:
			err = InvalidPacket
		}

		if err != nil {
			return
		}
	}
	err = InvalidPacket
	return
}

func doubleToAsciiTime(double_time float64) string {
	int_time := int(double_time)
	var days = math.Floor(double_time / (60 * 60 * 24))
	var hours = (int_time % (60 * 60 * 24)) / (60 * 60)
	var minutes = (int_time % (60 * 60)) / 60
	var seconds = (int_time % (60)) / 1
	var milliseconds = (int_time * 1000) % 1000
	var clock_time = (int_time * 12000) % 12

	return fmt.Sprintf("%03d%02d%02d%02d%03d%02d", days, hours, minutes, seconds, milliseconds, clock_time)
}

func asciiTimeToDouble(ascii_time []byte) (time float64, err error) {
	// Check time string format
	if len(asciiTime) != 16 {
		err = InvalidTime
	}
	_, err = strconv.Atoi(string(asciiTime))
	if err != nil {
		return
	}

	// Do the conversion now that we know it should work
	var ptr int = 0
	days, err := strconv.Atoi(string(ascii_time[ptr : ptr+3]))
	if err != nil {
		return
	}
	ptr += 3
	time += float64(60 * 60 * 24 * days)
	hours, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(60 * 60 * hours)
	minutes, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(60 * minutes)
	seconds, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(seconds)
	milliseconds, err := strconv.Atoi(string(ascii_time[ptr : ptr+3]))
	if err != nil {
		return
	}
	ptr += 3
	time += float64(milliseconds) / 1000.0
	clock, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(clock) / 12000.0
	return
}

func every(data []byte, check func(byte) bool) bool {
	for _, element := range data {
		if !check(element) {
			return false
		}
	}
	return true
}
<commit_msg>Fixed reference to undefined variable.<commit_after>package decoders

import (
	"errors"
	"fmt"
	"log"
	"math"
	"regexp"
	"strconv"

	"github.com/seadsystem/Backend/DB/landingzone/constants"
)

type SeadPacket struct {
	Type      byte
	Location  byte
	Timestamp float64
	Period    float64
	Count     int
	Data      float64
	Serial    int
}

var headerRegex *regexp.Regexp
var InvalidHeader = errors.New("Invalid header.")
var InvalidPacket = errors.New("Invalid packet.")
var InvalidTime = errors.New("Invalid time.")

func init() {
	var err error
	headerRegex, err = regexp.Compile(constants.HEADER_REGEX)
	if err != nil {
		log.Panic("Regex compile error:", err)
	}
}

func DecodeHeader(packet []byte) (serial int, err error) {
	serialStrings := headerRegex.FindSubmatch(packet)

	if serialStrings == nil {
		err = InvalidHeader
		return
	}
	log.Printf("Found %d serials.\n", len(serialStrings))
	if len(serialStrings) != 1 {
		err = InvalidHeader
		return
	}
	log.Printf("Header serial string: %s\n", string(serialStrings[0]))

	serial, err = strconv.Atoi(string(serialStrings[0]))
	return
}

// Extract data sent from sensor from request
func DecodePacket(buffer []byte) (packet SeadPacket, err error) {
	for i := 0; i < len(buffer); {
		datatype := buffer[i]
		i++

		// Switch on the type of data sent in the packet
		switch {
		case datatype == 'T':
			// Type
			packet.Type = buffer[i]
			i++
		case datatype == 'l':
			// Location
			packet.Location = buffer[i]
			i++
		case datatype == 't':
			// Timestamp
			packet.Timestamp, err = asciiTimeToDouble(buffer[i : i+14])
			i += 14
		case datatype == 'P':
			// Period separator
			packet.Period, err = asciiTimeToDouble(buffer[i : i+14])
			i += 14
		case datatype == 'C':
			// Count
			packet.Count, err = strconv.Atoi(string(buffer[i : i+2]))
			i += 2
		case datatype == 'D':
			// Data
			// if count isn't set, return error
			// TODO finish parsing data
		case datatype == 'S':
			// Serial
			packet.Serial, err = strconv.Atoi(string(buffer[i : i+6]))
			i += 6
		case datatype == 'X':
			return
		default:
			err = InvalidPacket
		}

		if err != nil {
			return
		}
	}
	err = InvalidPacket
	return
}

func doubleToAsciiTime(double_time float64) string {
	int_time := int(double_time)
	var days = math.Floor(double_time / (60 * 60 * 24))
	var hours = (int_time % (60 * 60 * 24)) / (60 * 60)
	var minutes = (int_time % (60 * 60)) / 60
	var seconds = (int_time % (60)) / 1
	var milliseconds = (int_time * 1000) % 1000
	var clock_time = (int_time * 12000) % 12

	return fmt.Sprintf("%03d%02d%02d%02d%03d%02d", days, hours, minutes, seconds, milliseconds, clock_time)
}

func asciiTimeToDouble(ascii_time []byte) (time float64, err error) {
	// Check time string format
	if len(ascii_time) != 16 {
		err = InvalidTime
	}
	_, err = strconv.Atoi(string(ascii_time))
	if err != nil {
		return
	}

	// Do the conversion now that we know it should work
	var ptr int = 0
	days, err := strconv.Atoi(string(ascii_time[ptr : ptr+3]))
	if err != nil {
		return
	}
	ptr += 3
	time += float64(60 * 60 * 24 * days)
	hours, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(60 * 60 * hours)
	minutes, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(60 * minutes)
	seconds, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(seconds)
	milliseconds, err := strconv.Atoi(string(ascii_time[ptr : ptr+3]))
	if err != nil {
		return
	}
	ptr += 3
	time += float64(milliseconds) / 1000.0
	clock, err := strconv.Atoi(string(ascii_time[ptr : ptr+2]))
	if err != nil {
		return
	}
	ptr += 2
	time += float64(clock) / 12000.0
	return
}

func every(data []byte, check func(byte) bool) bool {
	for _, element := range data {
		if !check(element) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"flag"
	"fmt"
	"image"
	"image/jpeg"
	"io"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/disintegration/imaging"
	"github.com/nfnt/resize"
	"github.com/rwcarlsen/goexif/exif"
)

var extensions = []string{".jpg", ".jpeg", ".JPG", ".JPEG"}
var metaDir = ".album"

var root = flag.String("root", "", "Album root")
var testMode = flag.Bool("test", false, "Test mode")
var httpAddress = flag.String("http", ":8080", "Default listening http address")

var metaRoot string

type FlipMode int

const (
	FlipVertical FlipMode = 1 << iota
	FlipHorizontal
)

const (
	topLeftSide     = 1
	topRightSide    = 2
	bottomRightSide = 3
	bottomLeftSide  = 4
	leftSideTop     = 5
	rightSideTop    = 6
	rightSideBottom = 7
	leftSideBottom  = 8
)

type PhotoTask struct {
	path     string
	root     string
	metaRoot string
	hash     string
}

type PhotoWalker struct {
	walkFunc func(path string, info os.FileInfo)
}

func (w PhotoWalker) photoWalker() filepath.WalkFunc {
	return func(path string, info os.FileInfo, err error) error {

		if info.IsDir() {

			if path == "." {
				return nil
			}

			if path == ".." {
				return nil
			}

			if strings.HasSuffix(path, ".app") {
				return filepath.SkipDir
			}

			if strings.HasSuffix(path, ".bundle") {
				return filepath.SkipDir
			}

			if strings.HasPrefix(info.Name(), ".") {
				return filepath.SkipDir
			}

			return nil
		}

		for _, ext := range extensions {
			if ext == filepath.Ext(info.Name()) {
				w.walkFunc(path, info)
				return nil
			}
		}

		return nil
	}
}

func HashPath(path string, root string, metaRoot string) string {
	normalizedPath := strings.TrimPrefix(path, root)
	normalizedPath = strings.TrimPrefix(normalizedPath, "/")

	hashPath := filepath.Join(metaRoot, "hash", normalizedPath) + ".sha1"
	return hashPath
}

func ThumbPath(hash string, root string, metaRoot string) string {
	hashPath := filepath.Join(metaRoot, "thumbs", hash) + ".jpg"
	return hashPath
}

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	flag.PrintDefaults()
}

func ExifOrientation(r io.Reader) (int, FlipMode) {
	var (
		angle int
		flip  FlipMode
	)

	ex, err := exif.Decode(r)
	if err != nil {
		log.Println("Could not decode Exif information")
		return 0, 0
	}

	tag, err := ex.Get(exif.Orientation)
	if err != nil {
		return 0, 0
	}

	orientation, err := tag.Int(0)

	if err != nil {
		return 0, 0
	}

	switch orientation {
	case topLeftSide:
		// no change required
	case topRightSide:
		flip = FlipHorizontal
	case bottomRightSide:
		angle = 180
	case bottomLeftSide:
		angle = 180
		flip = FlipHorizontal
	case leftSideTop:
		angle = -90
		flip = FlipHorizontal
	case rightSideTop:
		angle = -90
	case rightSideBottom:
		angle = 90
		flip = FlipHorizontal
	case leftSideBottom:
		angle = 90
	}

	return angle, flip
}

func main() {
	flag.Parse()

	if *root == "" {
		Usage()
		return
	}

	*root = filepath.Clean(*root)
	metaRoot = filepath.Join(*root, metaDir)

	if _, rootErr := ioutil.ReadDir(*root); rootErr != nil {
		log.Fatal("Root directory could not be read")
	}

	log.Println("Meta dir: " + metaDir)
	log.Println("Root: " + *root)

	workerCount := runtime.NumCPU()

	photos := []string{}
	hashes := make(map[string]string)

	photoWalker := PhotoWalker{}
	photoWalker.walkFunc = func(path string, info os.FileInfo) {
		photos = append(photos, path)
	}

	walkErr := filepath.Walk(*root, photoWalker.photoWalker())
	if walkErr != nil {
		log.Fatal(walkErr.Error())
	}

	fileCount := len(photos)
	counting := make(chan int)

	ticking := make(chan int, workerCount)
	finishing := make(chan bool)

	hashWorker := make(chan PhotoTask, workerCount)

	go func(work chan PhotoTask) {
		for task := range work {
			ticking <- 1

			task := task

			go func() {
				log.Println("Starting work on", task.path)
				file, fileErr := os.Open(task.path)

				if fileErr != nil {
					log.Println("Could not read photo file at", task.path, fileErr)
					<-ticking
					counting <- 1

					return
				}

				sha := sha1.New()
				io.Copy(sha, file)

				file.Close()

				sum := sha.Sum(nil)

				hashPath := HashPath(task.path, task.root, task.metaRoot)

				sumHex := hex.EncodeToString(sum)

				hashes[task.path] = sumHex

				if currentSumHex, hashErr := ioutil.ReadFile(hashPath); hashErr == nil {
					if string(currentSumHex) == sumHex {
						log.Println("Skipping", task.path)

						<-ticking
						counting <- 1

						return
					}
				}

				if !(*testMode) {
					os.MkdirAll(filepath.Dir(hashPath), 0755)
					file, fileErr := os.Create(hashPath)

					if fileErr != nil {
						log.Println("Could not create hash file: ", fileErr)
					} else {
						defer file.Close()

						if _, hashErr := file.WriteString(sumHex); hashErr != nil {
							log.Println("Could not write hash file: ", hashErr)
						}
					}
				}

				<-ticking
				counting <- 1

			}()

		}
	}(hashWorker)

	go func() {
		for _, path := range photos {
			task := PhotoTask{path: path, root: *root, metaRoot: metaRoot}
			hashWorker <- task
		}
	}()

	go func() {
		taskCount := 0

		for count := range counting {
			taskCount += count

			if taskCount == fileCount {
				close(counting)
				finishing <- true
				return
			}

		}
	}()

	<-finishing
	close(ticking)

	counting = make(chan int)

	ticking = make(chan int, workerCount)
	finishing = make(chan bool)

	photoWorker := make(chan PhotoTask, workerCount)

	go func(work chan PhotoTask) {
		for task := range work {
			ticking <- 1

			task := task

			go func() {
				file, fileErr := os.Open(task.path)

				if fileErr != nil {
					log.Println("Could not read photo file at", task.path, fileErr)
					<-ticking
					counting <- 1

					return
				}

				var exifBuffer bytes.Buffer

				img, imgErr := jpeg.Decode(io.TeeReader(file, &exifBuffer))

				file.Close()

				if imgErr != nil {
					log.Printf("Jpeg decode error: %v", imgErr)
					<-ticking
					counting <- 1
					return
				}

				maxDimension := 800.0
				width := float64(img.Bounds().Max.X - img.Bounds().Min.X)
				height := float64(img.Bounds().Max.Y - img.Bounds().Min.Y)

				var dstWidth float64
				var dstHeight float64

				biggerDimension := math.Max(width, height)

				if width < maxDimension && height < maxDimension {
					dstWidth = width
					dstHeight = height
				} else {
					scaleFactor := biggerDimension / maxDimension

					dstWidth = width / scaleFactor
					dstHeight = height / scaleFactor
				}

				udstWidth := uint(math.Ceil(dstWidth))
				udstHeight := uint(math.Ceil(dstHeight))

				var m image.Image
				if dstWidth == width && dstHeight == height {
					m = img
				} else {
					m = resize.Resize(udstWidth, udstHeight, img, resize.Bilinear)
				}

				angle, flip := ExifOrientation(&exifBuffer)

				switch angle {
				case 90:
					m = imaging.Rotate90(m)
				case -90:
					m = imaging.Rotate270(m)
				case 180:
					m = imaging.Rotate180(m)
				}

				switch flip {
				case FlipHorizontal:
					m = imaging.FlipH(m)
				case FlipVertical:
					m = imaging.FlipV(m)
				}

				thumbPath := ThumbPath(task.hash, task.root, task.metaRoot)

				if !(*testMode) {
					os.MkdirAll(filepath.Dir(thumbPath), 0755)

					thumbFile, thumbErr := os.Create(thumbPath)
					if thumbErr == nil {

						jpeg.Encode(thumbFile, m, &jpeg.Options{Quality: 85})

						thumbFile.Close()
					}
				}

				log.Printf("Processed image, %0.fx%0.f, thumbnail is %0.fx%0.f", width, height, dstWidth, dstHeight)
				<-ticking

				counting <- 1
			}()

		}
	}(photoWorker)

	go func() {
		for _, path := range photos {
			task := PhotoTask{path: path, root: *root, metaRoot: metaRoot, hash: hashes[path]}
			photoWorker <- task
		}
	}()

	go func() {
		taskCount := 0

		for count := range counting {
			taskCount += count

			if taskCount == fileCount {
				close(counting)
				finishing <- true
				return
			}

		}
	}()

	<-finishing
	close(ticking)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "<!doctype html>Hi, you have %d photos", len(photos))
	})

	log.Println("Listening on", *httpAddress)
	log.Fatal(http.ListenAndServe(*httpAddress, nil))
}
<commit_msg>Simplified code<commit_after>package main

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"flag"
	"fmt"
	"image/jpeg"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/disintegration/imaging"
	"github.com/nfnt/resize"
	"github.com/rwcarlsen/goexif/exif"
)

var extensions = []string{".jpg", ".jpeg", ".JPG", ".JPEG"}
var metaDir = ".album"

var root = flag.String("root", "", "Album root")
var testMode = flag.Bool("test", false, "Test mode")
var httpAddress = flag.String("http", ":8080", "Default listening http address")

var metaRoot string

type FlipMode int

const (
	FlipVertical FlipMode = 1 << iota
	FlipHorizontal
)

const (
	topLeftSide     = 1
	topRightSide    = 2
	bottomRightSide = 3
	bottomLeftSide  = 4
	leftSideTop     = 5
	rightSideTop    = 6
	rightSideBottom = 7
	leftSideBottom  = 8
)

type PhotoTask struct {
	path     string
	root     string
	metaRoot string
	hash     string
}

type PhotoWalker struct {
	walkFunc func(path string, info os.FileInfo)
}

func (w PhotoWalker) photoWalker() filepath.WalkFunc {
	return func(path string, info os.FileInfo, err error) error {

		if info.IsDir() {

			if path == "." {
				return nil
			}

			if path == ".." {
				return nil
			}

			if strings.HasSuffix(path, ".app") {
				return filepath.SkipDir
			}

			if strings.HasSuffix(path, ".bundle") {
				return filepath.SkipDir
			}

			if strings.HasPrefix(info.Name(), ".") {
				return filepath.SkipDir
			}

			return nil
		}

		for _, ext := range extensions {
			if ext == filepath.Ext(info.Name()) {
				w.walkFunc(path, info)
				return nil
			}
		}

		return nil
	}
}

func HashPath(path string, root string, metaRoot string) string {
	normalizedPath := strings.TrimPrefix(path, root)
	normalizedPath = strings.TrimPrefix(normalizedPath, "/")

	hashPath := filepath.Join(metaRoot, "hash", normalizedPath) + ".sha1"
	return hashPath
}

func ThumbPath(hash string, root string, metaRoot string) string {
	hashPath := filepath.Join(metaRoot, "thumbs", hash) + ".jpg"
	return hashPath
}

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	flag.PrintDefaults()
}

func ExifOrientation(r io.Reader) (int, FlipMode) {
	var (
		angle int
		flip  FlipMode
	)

	ex, err := exif.Decode(r)
	if err != nil {
		log.Println("Could not decode Exif information")
		return 0, 0
	}

	tag, err := ex.Get(exif.Orientation)
	if err != nil {
		return 0, 0
	}

	orientation, err := tag.Int(0)

	if err != nil {
		return 0, 0
	}

	switch orientation {
	case topLeftSide:
		// no change required
	case topRightSide:
		flip = FlipHorizontal
	case bottomRightSide:
		angle = 180
	case bottomLeftSide:
		angle = 180
		flip = FlipHorizontal
	case leftSideTop:
		angle = -90
		flip = FlipHorizontal
	case rightSideTop:
		angle = -90
	case rightSideBottom:
		angle = 90
		flip = FlipHorizontal
	case leftSideBottom:
		angle = 90
	}

	return angle, flip
}

func main() {
	flag.Parse()

	if *root == "" {
		Usage()
		return
	}

	*root = filepath.Clean(*root)
	metaRoot = filepath.Join(*root, metaDir)

	if _, rootErr := ioutil.ReadDir(*root); rootErr != nil {
		log.Fatal("Root directory could not be read")
	}

	log.Println("Meta dir: " + metaDir)
	log.Println("Root: " + *root)

	workerCount := runtime.NumCPU()

	photos := []string{}
	hashes := make(map[string]string)

	photoWalker := PhotoWalker{}
	photoWalker.walkFunc = func(path string, info os.FileInfo) {
		photos = append(photos, path)
	}

	walkErr := filepath.Walk(*root, photoWalker.photoWalker())
	if walkErr != nil {
		log.Fatal(walkErr.Error())
	}

	fileCount := len(photos)
	counting := make(chan int)

	ticking := make(chan int, workerCount)
	finishing := make(chan bool)

	hashWorker := make(chan PhotoTask, workerCount)

	go func(work chan PhotoTask) {
		for task := range work {
			ticking <- 1

			task := task

			go func() {
				log.Println("Starting work on", task.path)
				file, fileErr := os.Open(task.path)

				if fileErr != nil {
					log.Println("Could not read photo file at", task.path, fileErr)
					<-ticking
					counting <- 1

					return
				}

				sha := sha1.New()
				io.Copy(sha, file)

				file.Close()

				sum := sha.Sum(nil)

				hashPath := HashPath(task.path, task.root, task.metaRoot)

				sumHex := hex.EncodeToString(sum)

				hashes[task.path] = sumHex

				if currentSumHex, hashErr := ioutil.ReadFile(hashPath); hashErr == nil {
					if string(currentSumHex) == sumHex {
						log.Println("Skipping", task.path)

						<-ticking
						counting <- 1

						return
					}
				}

				if !(*testMode) {
					os.MkdirAll(filepath.Dir(hashPath), 0755)
					file, fileErr := os.Create(hashPath)

					if fileErr != nil {
						log.Println("Could not create hash file: ", fileErr)
					} else {
						defer file.Close()

						if _, hashErr := file.WriteString(sumHex); hashErr != nil {
							log.Println("Could not write hash file: ", hashErr)
						}
					}
				}

				<-ticking
				counting <- 1

			}()

		}
	}(hashWorker)

	go func() {
		for _, path := range photos {
			task := PhotoTask{path: path, root: *root, metaRoot: metaRoot}
			hashWorker <- task
		}
	}()

	go func() {
		taskCount := 0

		for count := range counting {
			taskCount += count

			if taskCount == fileCount {
				close(counting)
				finishing <- true
				return
			}

		}
	}()

	<-finishing
	close(ticking)

	counting = make(chan int)

	ticking = make(chan int, workerCount)
	finishing = make(chan bool)

	photoWorker := make(chan PhotoTask, workerCount)

	go func(work chan PhotoTask) {
		for task := range work {
			ticking <- 1

			task := task

			go func() {
				file, fileErr := os.Open(task.path)

				if fileErr != nil {
					log.Println("Could not read photo file at", task.path, fileErr)
					<-ticking
					counting <- 1

					return
				}

				var exifBuffer bytes.Buffer

				img, imgErr := jpeg.Decode(io.TeeReader(file, &exifBuffer))

				file.Close()

				if imgErr != nil {
					log.Printf("Jpeg decode error: %v", imgErr)
					<-ticking
					counting <- 1
					return
				}

				var maxDimension uint = 800
				m := resize.Thumbnail(maxDimension, maxDimension, img, resize.Bilinear)
				angle, flip := ExifOrientation(&exifBuffer)

				switch angle {
				case 90:
					m = imaging.Rotate90(m)
				case -90:
					m = imaging.Rotate270(m)
				case 180:
					m = imaging.Rotate180(m)
				}

				switch flip {
				case FlipHorizontal:
					m = imaging.FlipH(m)
				case FlipVertical:
					m = imaging.FlipV(m)
				}

				thumbPath := ThumbPath(task.hash, task.root, task.metaRoot)

				if !(*testMode) {
					os.MkdirAll(filepath.Dir(thumbPath), 0755)

					thumbFile, thumbErr := os.Create(thumbPath)
					if thumbErr == nil {

						jpeg.Encode(thumbFile, m, &jpeg.Options{Quality: 85})

						thumbFile.Close()
					}
				}

				log.Printf("Processed image, %dx%d, thumbnail is %dx%d", img.Bounds().Size().X, img.Bounds().Size().Y, m.Bounds().Size().X, m.Bounds().Size().Y)
				<-ticking

				counting <- 1
			}()

		}
	}(photoWorker)

	go func() {
		for _, path := range photos {
			task := PhotoTask{path: path, root: *root, metaRoot: metaRoot, hash: hashes[path]}
			photoWorker <- task
		}
	}()

	go func() {
		taskCount := 0

		for count := range counting {
			taskCount += count

			if taskCount == fileCount {
				close(counting)
				finishing <- true
				return
			}

		}
	}()

	<-finishing
	close(ticking)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "<!doctype html>Hi, you have %d photos", len(photos))
	})

	log.Println("Listening on", *httpAddress)
	log.Fatal(http.ListenAndServe(*httpAddress, nil))
}
<|endoftext|>


<commit_before>package godirwalk

import (
	"os"
	"path/filepath"
	"sort"

	"github.com/pkg/errors"
)

// Options provide parameters for how the Walk function operates.
type Options struct {
	// ErrorCallback specifies a function to be invoked in the case of an error
	// that could potentially be ignored while walking a file system
	// hierarchy. When set to nil or left as its zero-value, any error condition
	// causes Walk to immediately return the error describing what took
	// place. When non-nil, this user supplied function is invoked with the OS
	// pathname of the file system object that caused the error along with the
	// error that took place. The return value of the supplied ErrorCallback
	// function determines whether the error will cause Walk to halt immediately
	// as it would were no ErrorCallback value provided, or skip this file
	// system node yet continue on with the remaining nodes in the file system
	// hierarchy.
	//
	// ErrorCallback is invoked both for errors that are returned by the
	// runtime, and for errors returned by other user supplied callback
	// functions.
	ErrorCallback func(string, error) ErrorAction

	// FollowSymbolicLinks specifies whether Walk will follow symbolic links
	// that refer to directories. When set to false or left as its zero-value,
	// Walk will still invoke the callback function with symbolic link nodes,
	// but if the symbolic link refers to a directory, it will not recurse on
	// that directory. When set to true, Walk will recurse on symbolic links
	// that refer to a directory.
	FollowSymbolicLinks bool

	// Unsorted controls whether or not Walk will sort the immediate descendants
	// of a directory by their relative names prior to visiting each of those
	// entries.
	//
	// When set to false or left at its zero-value, Walk will get the list of
	// immediate descendants of a particular directory, sort that list by
	// lexical order of their names, and then visit each node in the list in
	// sorted order. This will cause Walk to always traverse the same directory
	// tree in the same order, however may be inefficient for directories with
	// many immediate descendants.
	//
	// When set to true, Walk skips sorting the list of immediate descendants
	// for a directory, and simply visits each node in the order the operating
	// system enumerated them. This will be more fast, but with the side effect
	// that the traversal order may be different from one invocation to the
	// next.
	Unsorted bool

	// Callback is a required function that Walk will invoke for every file
	// system node it encounters.
	Callback WalkFunc

	// PostChildrenCallback is an option function that Walk will invoke for
	// every file system directory it encounters after its children have been
	// processed.
	PostChildrenCallback WalkFunc

	// ScratchBuffer is an optional scratch buffer for Walk to use when reading
	// directory entries, to reduce amount of garbage generation. Not all
	// architectures take advantage of the scratch buffer.
	ScratchBuffer []byte
}

// ErrorAction defines a set of actions the Walk function could take based on
// the occurrence of an error while walking the file system. See the
// documentation for the ErrorCallback field of the Options structure for more
// information.
type ErrorAction int

const (
	// Halt is the ErrorAction return value when the upstream code wants to halt
	// the walk process when a runtime error takes place. It matches the default
	// action the Walk function would take were no ErrorCallback provided.
	Halt ErrorAction = iota

	// SkipNode is the ErrorAction return value when the upstream code wants to
	// ignore the runtime error for the current file system node, skip
	// processing of the node that caused the error, and continue walking the
	// file system hierarchy with the remaining nodes.
	SkipNode
)

// WalkFunc is the type of the function called for each file system node visited
// by Walk. The pathname argument will contain the argument to Walk as a prefix;
// that is, if Walk is called with "dir", which is a directory containing the
// file "a", the provided WalkFunc will be invoked with the argument "dir/a",
// using the correct os.PathSeparator for the Go Operating System architecture,
// GOOS. The directory entry argument is a pointer to a Dirent for the node,
// providing access to both the basename and the mode type of the file system
// node.
//
// If an error is returned by the Callback or PostChildrenCallback functions,
// and no ErrorCallback function is provided, processing stops. If an
// ErrorCallback function is provided, then it is invoked with the OS pathname
// of the node that caused the error along along with the error. The return
// value of the ErrorCallback function determines whether to halt processing, or
// skip this node and continue processing remaining file system nodes.
//
// The exception is when the function returns the special value
// filepath.SkipDir. If the function returns filepath.SkipDir when invoked on a
// directory, Walk skips the directory's contents entirely. If the function
// returns filepath.SkipDir when invoked on a non-directory file system node,
// Walk skips the remaining files in the containing directory. Note that any
// supplied ErrorCallback function is not invoked with filepath.SkipDir when the
// Callback or PostChildrenCallback functions return that special value.
type WalkFunc func(osPathname string, directoryEntry *Dirent) error

// Walk walks the file tree rooted at the specified directory, calling the
// specified callback function for each file system node in the tree, including
// root, symbolic links, and other node types. The nodes are walked in lexical
// order, which makes the output deterministic but means that for very large
// directories this function can be inefficient.
//
// This function is often much faster than filepath.Walk because it does not
// invoke os.Stat for every node it encounters, but rather obtains the file
// system node type when it reads the parent directory.
//
// If a runtime error occurs, either from the operating system or from the
// upstream Callback or PostChildrenCallback functions, processing typically
// halts. However, when an ErrorCallback function is provided in the provided
// Options structure, that function is invoked with the error along with the OS
// pathname of the file system node that caused the error. The ErrorCallback
// function's return value determines the action that Walk will then take.
//
//    func main() {
//        dirname := "."
//        if len(os.Args) > 1 {
//            dirname = os.Args[1]
//        }
//        err := godirwalk.Walk(dirname, &godirwalk.Options{
//            Callback: func(osPathname string, de *godirwalk.Dirent) error {
//                fmt.Printf("%s %s\n", de.ModeType(), osPathname)
//                return nil
//            },
//            ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction {
//            	// Your program may want to log the error somehow.
//            	fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
//
//            	// For the purposes of this example, a simple SkipNode will suffice,
//            	// although in reality perhaps additional logic might be called for.
//            	return godirwalk.SkipNode
//            },
//        })
//        if err != nil {
//            fmt.Fprintf(os.Stderr, "%s\n", err)
//            os.Exit(1)
//        }
//    }
func Walk(pathname string, options *Options) error {
	pathname = filepath.Clean(pathname)

	var fi os.FileInfo
	var err error

	if options.FollowSymbolicLinks {
		fi, err = os.Stat(pathname)
		if err != nil {
			return errors.Wrap(err, "cannot Stat")
		}
	} else {
		fi, err = os.Lstat(pathname)
		if err != nil {
			return errors.Wrap(err, "cannot Lstat")
		}
	}

	mode := fi.Mode()
	if mode&os.ModeDir == 0 {
		return errors.Errorf("cannot Walk non-directory: %s", pathname)
	}

	dirent := &Dirent{
		name:     filepath.Base(pathname),
		modeType: mode & os.ModeType,
	}

	// If ErrorCallback is nil, set to a default value that halts the walk
	// process on all operating system errors. This is done to allow error
	// handling to be more succinct in the walk code.
	if options.ErrorCallback == nil {
		options.ErrorCallback = defaultErrorCallback
	}

	err = walk(pathname, dirent, options)
	if err == filepath.SkipDir {
		return nil // silence SkipDir for top level
	}
	return err
}

// defaultErrorCallback always returns Halt because if the upstream code did not
// provide an ErrorCallback function, walking the file system hierarchy ought to
// halt upon any operating system error.
func defaultErrorCallback(_ string, _ error) ErrorAction { return Halt }

// walk recursively traverses the file system node specified by pathname and the
// Dirent.
func walk(osPathname string, dirent *Dirent, options *Options) error {
	err := options.Callback(osPathname, dirent)
	if err != nil {
		if err == filepath.SkipDir {
			return err
		}
		err = errors.Wrap(err, "Callback") // wrap potential errors returned by callback
		if action := options.ErrorCallback(osPathname, err); action == SkipNode {
			return nil
		}
		return err
	}

	// On some platforms, an entry can have more than one mode type bit set.
	// For instance, it could have both the symlink bit and the directory bit
	// set indicating it's a symlink to a directory.
	if dirent.IsSymlink() {
		if !options.FollowSymbolicLinks {
			return nil
		}
		// Only need to Stat entry if platform did not already have os.ModeDir
		// set, such as would be the case for unix like operating systems. (This
		// guard eliminates extra os.Stat check on Windows.)
		if !dirent.IsDir() {
			referent, err := os.Readlink(osPathname)
			if err != nil {
				err = errors.Wrap(err, "cannot Readlink")
				if action := options.ErrorCallback(osPathname, err); action == SkipNode {
					return nil
				}
				return err
			}

			osp := filepath.Join(filepath.Dir(osPathname), referent)
			fi, err := os.Stat(osp)
			if err != nil {
				err = errors.Wrap(err, "cannot Stat")
				if action := options.ErrorCallback(osp, err); action == SkipNode {
					return nil
				}
				return err
			}
			dirent.modeType = fi.Mode() & os.ModeType
		}
	}

	if !dirent.IsDir() {
		return nil
	}

	// If get here, then specified pathname refers to a directory.
	deChildren, err := ReadDirents(osPathname, options.ScratchBuffer)
	if err != nil {
		err = errors.Wrap(err, "cannot ReadDirents")
		if action := options.ErrorCallback(osPathname, err); action == SkipNode {
			return nil
		}
		return err
	}

	if !options.Unsorted {
		sort.Sort(deChildren) // sort children entries unless upstream says to leave unsorted
	}

	for _, deChild := range deChildren {
		osChildname := filepath.Join(osPathname, deChild.name)
		err = walk(osChildname, deChild, options)
		if err != nil {
			if err != filepath.SkipDir {
				return err
			}
			// If received skipdir on a directory, stop processing that
			// directory, but continue to its siblings. If received skipdir on a
			// non-directory, stop processing remaining siblings.
			if deChild.IsSymlink() {
				// Only need to Stat entry if platform did not already have
				// os.ModeDir set, such as would be the case for unix like
				// operating systems. (This guard eliminates extra os.Stat check
				// on Windows.)
				if !deChild.IsDir() {
					// Resolve symbolic link referent to determine whether node
					// is directory or not.
					referent, err := os.Readlink(osChildname)
					if err != nil {
						err = errors.Wrap(err, "cannot Readlink")
						if action := options.ErrorCallback(osChildname, err); action == SkipNode {
							continue // with next child
						}
						return err
					}
					osp := filepath.Join(osPathname, referent)
					fi, err := os.Stat(osp)
					if err != nil {
						err = errors.Wrap(err, "cannot Stat")
						if action := options.ErrorCallback(osp, err); action == SkipNode {
							continue // with next child
						}
						return err
					}
					deChild.modeType = fi.Mode() & os.ModeType
				}
			}
			if !deChild.IsDir() {
				// If not directory, return immediately, thus skipping remainder
				// of siblings.
				return nil
			}
		}
	}

	if options.PostChildrenCallback == nil {
		return nil
	}

	err = options.PostChildrenCallback(osPathname, dirent)
	if err == nil || err == filepath.SkipDir {
		return err
	}

	err = errors.Wrap(err, "PostChildrenCallback") // wrap potential errors returned by callback
	if action := options.ErrorCallback(osPathname, err); action == SkipNode {
		return nil
	}
	return err
}
<commit_msg>uses 64 KiB scratch buffer if not provided one<commit_after>package godirwalk

import (
	"os"
	"path/filepath"
	"sort"

	"github.com/pkg/errors"
)

// Options provide parameters for how the Walk function operates.
type Options struct {
	// ErrorCallback specifies a function to be invoked in the case of an error
	// that could potentially be ignored while walking a file system
	// hierarchy. When set to nil or left as its zero-value, any error condition
	// causes Walk to immediately return the error describing what took
	// place. When non-nil, this user supplied function is invoked with the OS
	// pathname of the file system object that caused the error along with the
	// error that took place. The return value of the supplied ErrorCallback
	// function determines whether the error will cause Walk to halt immediately
	// as it would were no ErrorCallback value provided, or skip this file
	// system node yet continue on with the remaining nodes in the file system
	// hierarchy.
	//
	// ErrorCallback is invoked both for errors that are returned by the
	// runtime, and for errors returned by other user supplied callback
	// functions.
	ErrorCallback func(string, error) ErrorAction

	// FollowSymbolicLinks specifies whether Walk will follow symbolic links
	// that refer to directories. When set to false or left as its zero-value,
	// Walk will still invoke the callback function with symbolic link nodes,
	// but if the symbolic link refers to a directory, it will not recurse on
	// that directory. When set to true, Walk will recurse on symbolic links
	// that refer to a directory.
	FollowSymbolicLinks bool

	// Unsorted controls whether or not Walk will sort the immediate descendants
	// of a directory by their relative names prior to visiting each of those
	// entries.
	//
	// When set to false or left at its zero-value, Walk will get the list of
	// immediate descendants of a particular directory, sort that list by
	// lexical order of their names, and then visit each node in the list in
	// sorted order. This will cause Walk to always traverse the same directory
	// tree in the same order, however may be inefficient for directories with
	// many immediate descendants.
	//
	// When set to true, Walk skips sorting the list of immediate descendants
	// for a directory, and simply visits each node in the order the operating
	// system enumerated them. This will be more fast, but with the side effect
	// that the traversal order may be different from one invocation to the
	// next.
	Unsorted bool

	// Callback is a required function that Walk will invoke for every file
	// system node it encounters.
	Callback WalkFunc

	// PostChildrenCallback is an option function that Walk will invoke for
	// every file system directory it encounters after its children have been
	// processed.
	PostChildrenCallback WalkFunc

	// ScratchBuffer is an optional scratch buffer for Walk to use when reading
	// directory entries, to reduce amount of garbage generation. Not all
	// architectures take advantage of the scratch buffer.
	ScratchBuffer []byte
}

// ErrorAction defines a set of actions the Walk function could take based on
// the occurrence of an error while walking the file system. See the
// documentation for the ErrorCallback field of the Options structure for more
// information.
type ErrorAction int

const (
	// Halt is the ErrorAction return value when the upstream code wants to halt
	// the walk process when a runtime error takes place. It matches the default
	// action the Walk function would take were no ErrorCallback provided.
	Halt ErrorAction = iota

	// SkipNode is the ErrorAction return value when the upstream code wants to
	// ignore the runtime error for the current file system node, skip
	// processing of the node that caused the error, and continue walking the
	// file system hierarchy with the remaining nodes.
	SkipNode
)

// WalkFunc is the type of the function called for each file system node visited
// by Walk. The pathname argument will contain the argument to Walk as a prefix;
// that is, if Walk is called with "dir", which is a directory containing the
// file "a", the provided WalkFunc will be invoked with the argument "dir/a",
// using the correct os.PathSeparator for the Go Operating System architecture,
// GOOS. The directory entry argument is a pointer to a Dirent for the node,
// providing access to both the basename and the mode type of the file system
// node.
//
// If an error is returned by the Callback or PostChildrenCallback functions,
// and no ErrorCallback function is provided, processing stops. If an
// ErrorCallback function is provided, then it is invoked with the OS pathname
// of the node that caused the error along along with the error. The return
// value of the ErrorCallback function determines whether to halt processing, or
// skip this node and continue processing remaining file system nodes.
//
// The exception is when the function returns the special value
// filepath.SkipDir. If the function returns filepath.SkipDir when invoked on a
// directory, Walk skips the directory's contents entirely. If the function
// returns filepath.SkipDir when invoked on a non-directory file system node,
// Walk skips the remaining files in the containing directory. Note that any
// supplied ErrorCallback function is not invoked with filepath.SkipDir when the
// Callback or PostChildrenCallback functions return that special value.
type WalkFunc func(osPathname string, directoryEntry *Dirent) error

// Walk walks the file tree rooted at the specified directory, calling the
// specified callback function for each file system node in the tree, including
// root, symbolic links, and other node types. The nodes are walked in lexical
// order, which makes the output deterministic but means that for very large
// directories this function can be inefficient.
//
// This function is often much faster than filepath.Walk because it does not
// invoke os.Stat for every node it encounters, but rather obtains the file
// system node type when it reads the parent directory.
//
// If a runtime error occurs, either from the operating system or from the
// upstream Callback or PostChildrenCallback functions, processing typically
// halts. However, when an ErrorCallback function is provided in the provided
// Options structure, that function is invoked with the error along with the OS
// pathname of the file system node that caused the error. The ErrorCallback
// function's return value determines the action that Walk will then take.
//
//    func main() {
//        dirname := "."
//        if len(os.Args) > 1 {
//            dirname = os.Args[1]
//        }
//        err := godirwalk.Walk(dirname, &godirwalk.Options{
//            Callback: func(osPathname string, de *godirwalk.Dirent) error {
//                fmt.Printf("%s %s\n", de.ModeType(), osPathname)
//                return nil
//            },
//            ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction {
//            	// Your program may want to log the error somehow.
//            	fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
//
//            	// For the purposes of this example, a simple SkipNode will suffice,
//            	// although in reality perhaps additional logic might be called for.
//            	return godirwalk.SkipNode
//            },
//        })
//        if err != nil {
//            fmt.Fprintf(os.Stderr, "%s\n", err)
//            os.Exit(1)
//        }
//    }
func Walk(pathname string, options *Options) error {
	pathname = filepath.Clean(pathname)

	var fi os.FileInfo
	var err error

	if options.FollowSymbolicLinks {
		fi, err = os.Stat(pathname)
		if err != nil {
			return errors.Wrap(err, "cannot Stat")
		}
	} else {
		fi, err = os.Lstat(pathname)
		if err != nil {
			return errors.Wrap(err, "cannot Lstat")
		}
	}

	mode := fi.Mode()
	if mode&os.ModeDir == 0 {
		return errors.Errorf("cannot Walk non-directory: %s", pathname)
	}

	dirent := &Dirent{
		name:     filepath.Base(pathname),
		modeType: mode & os.ModeType,
	}

	// If ErrorCallback is nil, set to a default value that halts the walk
	// process on all operating system errors. This is done to allow error
	// handling to be more succinct in the walk code.
	if options.ErrorCallback == nil {
		options.ErrorCallback = defaultErrorCallback
	}

	if options.ScratchBuffer == nil {
		options.ScratchBuffer = make([]byte, 64*1024)
	}

	err = walk(pathname, dirent, options)
	if err == filepath.SkipDir {
		return nil // silence SkipDir for top level
	}
	return err
}

// defaultErrorCallback always returns Halt because if the upstream code did not
// provide an ErrorCallback function, walking the file system hierarchy ought to
// halt upon any operating system error.
func defaultErrorCallback(_ string, _ error) ErrorAction { return Halt }

// walk recursively traverses the file system node specified by pathname and the
// Dirent.
func walk(osPathname string, dirent *Dirent, options *Options) error {
	err := options.Callback(osPathname, dirent)
	if err != nil {
		if err == filepath.SkipDir {
			return err
		}
		err = errors.Wrap(err, "Callback") // wrap potential errors returned by callback
		if action := options.ErrorCallback(osPathname, err); action == SkipNode {
			return nil
		}
		return err
	}

	// On some platforms, an entry can have more than one mode type bit set.
	// For instance, it could have both the symlink bit and the directory bit
	// set indicating it's a symlink to a directory.
	if dirent.IsSymlink() {
		if !options.FollowSymbolicLinks {
			return nil
		}
		// Only need to Stat entry if platform did not already have os.ModeDir
		// set, such as would be the case for unix like operating systems. (This
		// guard eliminates extra os.Stat check on Windows.)
		if !dirent.IsDir() {
			referent, err := os.Readlink(osPathname)
			if err != nil {
				err = errors.Wrap(err, "cannot Readlink")
				if action := options.ErrorCallback(osPathname, err); action == SkipNode {
					return nil
				}
				return err
			}

			osp := filepath.Join(filepath.Dir(osPathname), referent)
			fi, err := os.Stat(osp)
			if err != nil {
				err = errors.Wrap(err, "cannot Stat")
				if action := options.ErrorCallback(osp, err); action == SkipNode {
					return nil
				}
				return err
			}
			dirent.modeType = fi.Mode() & os.ModeType
		}
	}

	if !dirent.IsDir() {
		return nil
	}

	// If get here, then specified pathname refers to a directory.
	deChildren, err := ReadDirents(osPathname, options.ScratchBuffer)
	if err != nil {
		err = errors.Wrap(err, "cannot ReadDirents")
		if action := options.ErrorCallback(osPathname, err); action == SkipNode {
			return nil
		}
		return err
	}

	if !options.Unsorted {
		sort.Sort(deChildren) // sort children entries unless upstream says to leave unsorted
	}

	for _, deChild := range deChildren {
		osChildname := filepath.Join(osPathname, deChild.name)
		err = walk(osChildname, deChild, options)
		if err != nil {
			if err != filepath.SkipDir {
				return err
			}
			// If received skipdir on a directory, stop processing that
			// directory, but continue to its siblings. If received skipdir on a
			// non-directory, stop processing remaining siblings.
			if deChild.IsSymlink() {
				// Only need to Stat entry if platform did not already have
				// os.ModeDir set, such as would be the case for unix like
				// operating systems. (This guard eliminates extra os.Stat check
				// on Windows.)
				if !deChild.IsDir() {
					// Resolve symbolic link referent to determine whether node
					// is directory or not.
					referent, err := os.Readlink(osChildname)
					if err != nil {
						err = errors.Wrap(err, "cannot Readlink")
						if action := options.ErrorCallback(osChildname, err); action == SkipNode {
							continue // with next child
						}
						return err
					}
					osp := filepath.Join(osPathname, referent)
					fi, err := os.Stat(osp)
					if err != nil {
						err = errors.Wrap(err, "cannot Stat")
						if action := options.ErrorCallback(osp, err); action == SkipNode {
							continue // with next child
						}
						return err
					}
					deChild.modeType = fi.Mode() & os.ModeType
				}
			}
			if !deChild.IsDir() {
				// If not directory, return immediately, thus skipping remainder
				// of siblings.
				return nil
			}
		}
	}

	if options.PostChildrenCallback == nil {
		return nil
	}

	err = options.PostChildrenCallback(osPathname, dirent)
	if err == nil || err == filepath.SkipDir {
		return err
	}

	err = errors.Wrap(err, "PostChildrenCallback") // wrap potential errors returned by callback
	if action := options.ErrorCallback(osPathname, err); action == SkipNode {
		return nil
	}
	return err
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"time"
)

type obj map[string]interface{}

// dash runs the given method and command on the dashboard.
// If args is non-nil it is encoded as the URL query string.
// If req is non-nil it is JSON-encoded and passed as the body of the HTTP POST.
// If resp is non-nil the server's response is decoded into the value pointed
// to by resp (resp must be a pointer).
func dash(meth, cmd string, args url.Values, req, resp interface{}) error {
	var r *http.Response
	var err error
	if *verbose {
		log.Println("dash", meth, cmd, args, req)
	}
	cmd = "http://" + *dashboard + "/" + cmd
	if len(args) > 0 {
		cmd += "?" + args.Encode()
	}
	switch meth {
	case "GET":
		if req != nil {
			log.Panicf("%s to %s with req", meth, cmd)
		}
		r, err = http.Get(cmd)
	case "POST":
		var body io.Reader
		if req != nil {
			b, err := json.Marshal(req)
			if err != nil {
				return err
			}
			body = bytes.NewBuffer(b)
		}
		r, err = http.Post(cmd, "text/json", body)
	default:
		log.Panicf("%s: invalid method %q", cmd, meth)
		panic("invalid method: " + meth)
	}
	if err != nil {
		return err
	}

	defer r.Body.Close()
	body := new(bytes.Buffer)
	if _, err := body.ReadFrom(r.Body); err != nil {
		return err
	}

	// Read JSON-encoded Response into provided resp
	// and return an error if present.
	var result = struct {
		Response interface{}
		Error    string
	}{
		// Put the provided resp in here as it can be a pointer to
		// some value we should unmarshal into.
		Response: resp,
	}
	if err = json.Unmarshal(body.Bytes(), &result); err != nil {
		log.Printf("json unmarshal %#q: %s\n", body.Bytes(), err)
		return err
	}
	if result.Error != "" {
		return errors.New(result.Error)
	}

	return nil
}

// todo returns the next hash to build.
func (b *Builder) todo(kind, pkg, goHash string) (rev string, err error) {
	args := url.Values{
		"kind":        {kind},
		"builder":     {b.name},
		"packagePath": {pkg},
		"goHash":      {goHash},
	}
	var resp *struct {
		Kind string
		Data struct {
			Hash string
		}
	}
	if err = dash("GET", "todo", args, nil, &resp); err != nil {
		return "", err
	}
	if resp == nil {
		return "", nil
	}
	if kind != resp.Kind {
		return "", fmt.Errorf("expecting Kind %q, got %q", kind, resp.Kind)
	}
	return resp.Data.Hash, nil
}

// recordResult sends build results to the dashboard
func (b *Builder) recordResult(ok bool, pkg, hash, goHash, buildLog string, runTime time.Duration) error {
	req := obj{
		"Builder":     b.name,
		"PackagePath": pkg,
		"Hash":        hash,
		"GoHash":      goHash,
		"OK":          ok,
		"Log":         buildLog,
		"RunTime":     runTime,
	}
	args := url.Values{"key": {b.key}, "builder": {b.name}}
	return dash("POST", "result", args, req, nil)
}

// packages fetches a list of package paths from the dashboard
func packages() (pkgs []string, err error) {
	return nil, nil
	/* TODO(adg): un-stub this once the new package builder design is done
	var resp struct {
		Packages []struct {
			Path string
		}
	}
	err = dash("GET", "package", &resp, param{"fmt": "json"})
	if err != nil {
		return
	}
	for _, p := range resp.Packages {
		pkgs = append(pkgs, p.Path)
	}
	return
	*/
}

// updatePackage sends package build results and info dashboard
func (b *Builder) updatePackage(pkg string, ok bool, buildLog, info string) error {
	return nil
	/* TODO(adg): un-stub this once the new package builder design is done
	return dash("POST", "package", nil, param{
		"builder": b.name,
		"key":     b.key,
		"path":    pkg,
		"ok":      strconv.FormatBool(ok),
		"log":     buildLog,
		"info":    info,
	})
	*/
}

func postCommit(key, pkg string, l *HgLog) error {
	t, err := time.Parse(time.RFC3339, l.Date)
	if err != nil {
		return fmt.Errorf("parsing %q: %v", l.Date, t)
	}
	return dash("POST", "commit", url.Values{"key": {key}}, obj{
		"PackagePath": pkg,
		"Hash":        l.Hash,
		"ParentHash":  l.Parent,
		"Time":        t.Unix() * 1e6, // in microseconds, yuck!
		"User":        l.Author,
		"Desc":        l.Desc,
	}, nil)
}

func dashboardCommit(pkg, hash string) bool {
	err := dash("GET", "commit", url.Values{
		"packagePath": {pkg},
		"hash":        {hash},
	}, nil, nil)
	return err == nil
}

func dashboardPackages() []string {
	var resp []struct {
		Path string
	}
	if err := dash("GET", "packages", nil, nil, &resp); err != nil {
		log.Println("dashboardPackages:", err)
		return nil
	}
	var pkgs []string
	for _, r := range resp {
		pkgs = append(pkgs, r.Path)
	}
	return pkgs
}
<commit_msg>misc/dashboard/builder: fix comment in http.go<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"time"
)

type obj map[string]interface{}

// dash runs the given method and command on the dashboard.
// If args is non-nil it is encoded as the URL query string.
// If req is non-nil it is JSON-encoded and passed as the body of the HTTP POST.
// If resp is non-nil the server's response is decoded into the value pointed
// to by resp (resp must be a pointer).
func dash(meth, cmd string, args url.Values, req, resp interface{}) error {
	var r *http.Response
	var err error
	if *verbose {
		log.Println("dash", meth, cmd, args, req)
	}
	cmd = "http://" + *dashboard + "/" + cmd
	if len(args) > 0 {
		cmd += "?" + args.Encode()
	}
	switch meth {
	case "GET":
		if req != nil {
			log.Panicf("%s to %s with req", meth, cmd)
		}
		r, err = http.Get(cmd)
	case "POST":
		var body io.Reader
		if req != nil {
			b, err := json.Marshal(req)
			if err != nil {
				return err
			}
			body = bytes.NewBuffer(b)
		}
		r, err = http.Post(cmd, "text/json", body)
	default:
		log.Panicf("%s: invalid method %q", cmd, meth)
		panic("invalid method: " + meth)
	}
	if err != nil {
		return err
	}

	defer r.Body.Close()
	body := new(bytes.Buffer)
	if _, err := body.ReadFrom(r.Body); err != nil {
		return err
	}

	// Read JSON-encoded Response into provided resp
	// and return an error if present.
	var result = struct {
		Response interface{}
		Error    string
	}{
		// Put the provided resp in here as it can be a pointer to
		// some value we should unmarshal into.
		Response: resp,
	}
	if err = json.Unmarshal(body.Bytes(), &result); err != nil {
		log.Printf("json unmarshal %#q: %s\n", body.Bytes(), err)
		return err
	}
	if result.Error != "" {
		return errors.New(result.Error)
	}

	return nil
}

// todo returns the next hash to build.
func (b *Builder) todo(kind, pkg, goHash string) (rev string, err error) {
	args := url.Values{
		"kind":        {kind},
		"builder":     {b.name},
		"packagePath": {pkg},
		"goHash":      {goHash},
	}
	var resp *struct {
		Kind string
		Data struct {
			Hash string
		}
	}
	if err = dash("GET", "todo", args, nil, &resp); err != nil {
		return "", err
	}
	if resp == nil {
		return "", nil
	}
	if kind != resp.Kind {
		return "", fmt.Errorf("expecting Kind %q, got %q", kind, resp.Kind)
	}
	return resp.Data.Hash, nil
}

// recordResult sends build results to the dashboard
func (b *Builder) recordResult(ok bool, pkg, hash, goHash, buildLog string, runTime time.Duration) error {
	req := obj{
		"Builder":     b.name,
		"PackagePath": pkg,
		"Hash":        hash,
		"GoHash":      goHash,
		"OK":          ok,
		"Log":         buildLog,
		"RunTime":     runTime,
	}
	args := url.Values{"key": {b.key}, "builder": {b.name}}
	return dash("POST", "result", args, req, nil)
}

// packages fetches a list of package paths from the dashboard
func packages() (pkgs []string, err error) {
	return nil, nil
	/* TODO(adg): un-stub this once the new package builder design is done
	var resp struct {
		Packages []struct {
			Path string
		}
	}
	err = dash("GET", "package", &resp, param{"fmt": "json"})
	if err != nil {
		return
	}
	for _, p := range resp.Packages {
		pkgs = append(pkgs, p.Path)
	}
	return
	*/
}

// updatePackage sends package build results and info to the dashboard
func (b *Builder) updatePackage(pkg string, ok bool, buildLog, info string) error {
	return nil
	/* TODO(adg): un-stub this once the new package builder design is done
	return dash("POST", "package", nil, param{
		"builder": b.name,
		"key":     b.key,
		"path":    pkg,
		"ok":      strconv.FormatBool(ok),
		"log":     buildLog,
		"info":    info,
	})
	*/
}

func postCommit(key, pkg string, l *HgLog) error {
	t, err := time.Parse(time.RFC3339, l.Date)
	if err != nil {
		return fmt.Errorf("parsing %q: %v", l.Date, t)
	}
	return dash("POST", "commit", url.Values{"key": {key}}, obj{
		"PackagePath": pkg,
		"Hash":        l.Hash,
		"ParentHash":  l.Parent,
		"Time":        t.Unix() * 1e6, // in microseconds, yuck!
		"User":        l.Author,
		"Desc":        l.Desc,
	}, nil)
}

func dashboardCommit(pkg, hash string) bool {
	err := dash("GET", "commit", url.Values{
		"packagePath": {pkg},
		"hash":        {hash},
	}, nil, nil)
	return err == nil
}

func dashboardPackages() []string {
	var resp []struct {
		Path string
	}
	if err := dash("GET", "packages", nil, nil, &resp); err != nil {
		log.Println("dashboardPackages:", err)
		return nil
	}
	var pkgs []string
	for _, r := range resp {
		pkgs = append(pkgs, r.Path)
	}
	return pkgs
}
<|endoftext|>