<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

type Etcd2 struct {
	AdvertiseClientURLs      string `yaml:"advertise_client_urls"         env:"ETCD_ADVERTISE_CLIENT_URLS"`
	CAFile                   string `yaml:"ca_file"                       env:"ETCD_CA_FILE"`
	CertFile                 string `yaml:"cert_file"                     env:"ETCD_CERT_FILE"`
	CorsOrigins              string `yaml:"cors"                          env:"ETCD_CORS"`
	DataDir                  string `yaml:"data_dir"                      env:"ETCD_DATA_DIR"`
	Discovery                string `yaml:"discovery"                     env:"ETCD_DISCOVERY"`
	DiscoveryFallback        string `yaml:"discovery_fallback"            env:"ETCD_DISCOVERY_FALLBACK"`
	DiscoverySRV             string `yaml:"discovery_srv"                 env:"ETCD_DISCOVERY_SRV"`
	DiscoveryProxy           string `yaml:"discovery_proxy"               env:"ETCD_DISCOVERY_PROXY"`
	ElectionTimeout          int    `yaml:"election_timeout"              env:"ETCD_ELECTION_TIMEOUT"`
	HeartbeatInterval        int    `yaml:"heartbeat_interval"            env:"ETCD_HEARTBEAT_INTERVAL"`
	InitialAdvertisePeerURLs string `yaml:"initial_advertise_peer_urls"   env:"ETCD_INITIAL_ADVERTISE_PEER_URLS"`
	InitialCluster           string `yaml:"initial_cluster"               env:"ETCD_INITIAL_CLUSTER"`
	InitialClusterState      string `yaml:"initial_cluster_state"         env:"ETCD_INITIAL_CLUSTER_STATE"`
	InitialClusterToken      string `yaml:"initial_cluster_token"         env:"ETCD_INITIAL_CLUSTER_TOKEN"`
	KeyFile                  string `yaml:"key_file"                      env:"ETCD_KEY_FILE"`
	ListenClientURLs         string `yaml:"listen_client_urls"            env:"ETCD_LISTEN_CLIENT_URLS"`
	ListenPeerURLs           string `yaml:"listen_peer_urls"              env:"ETCD_LISTEN_PEER_URLS"`
	MaxSnapshots             int    `yaml:"max_snapshots"                 env:"ETCD_MAX_SNAPSHOTS"`
	MaxWALs                  int    `yaml:"max_wals"                      env:"ETCD_MAX_WALS"`
	Name                     string `yaml:"name"                          env:"ETCD_NAME"`
	PeerCAFile               string `yaml:"peer_ca_file"                  env:"ETCD_PEER_CA_FILE"`
	PeerCertFile             string `yaml:"peer_cert_file"                env:"ETCD_PEER_CERT_FILE"`
	PeerKeyFile              string `yaml:"peer_key_file"                 env:"ETCD_PEER_KEY_FILE"`
	Proxy                    string `yaml:"proxy"                         env:"ETCD_PROXY"`
	SnapshotCount            int    `yaml:"snapshot_count"                env:"ETCD_SNAPSHOTCOUNT"`
}
<commit_msg>config: specific valid values for ETCD_PROXY<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

type Etcd2 struct {
	AdvertiseClientURLs      string `yaml:"advertise_client_urls"         env:"ETCD_ADVERTISE_CLIENT_URLS"`
	CAFile                   string `yaml:"ca_file"                       env:"ETCD_CA_FILE"`
	CertFile                 string `yaml:"cert_file"                     env:"ETCD_CERT_FILE"`
	CorsOrigins              string `yaml:"cors"                          env:"ETCD_CORS"`
	DataDir                  string `yaml:"data_dir"                      env:"ETCD_DATA_DIR"`
	Discovery                string `yaml:"discovery"                     env:"ETCD_DISCOVERY"`
	DiscoveryFallback        string `yaml:"discovery_fallback"            env:"ETCD_DISCOVERY_FALLBACK"`
	DiscoverySRV             string `yaml:"discovery_srv"                 env:"ETCD_DISCOVERY_SRV"`
	DiscoveryProxy           string `yaml:"discovery_proxy"               env:"ETCD_DISCOVERY_PROXY"`
	ElectionTimeout          int    `yaml:"election_timeout"              env:"ETCD_ELECTION_TIMEOUT"`
	HeartbeatInterval        int    `yaml:"heartbeat_interval"            env:"ETCD_HEARTBEAT_INTERVAL"`
	InitialAdvertisePeerURLs string `yaml:"initial_advertise_peer_urls"   env:"ETCD_INITIAL_ADVERTISE_PEER_URLS"`
	InitialCluster           string `yaml:"initial_cluster"               env:"ETCD_INITIAL_CLUSTER"`
	InitialClusterState      string `yaml:"initial_cluster_state"         env:"ETCD_INITIAL_CLUSTER_STATE"`
	InitialClusterToken      string `yaml:"initial_cluster_token"         env:"ETCD_INITIAL_CLUSTER_TOKEN"`
	KeyFile                  string `yaml:"key_file"                      env:"ETCD_KEY_FILE"`
	ListenClientURLs         string `yaml:"listen_client_urls"            env:"ETCD_LISTEN_CLIENT_URLS"`
	ListenPeerURLs           string `yaml:"listen_peer_urls"              env:"ETCD_LISTEN_PEER_URLS"`
	MaxSnapshots             int    `yaml:"max_snapshots"                 env:"ETCD_MAX_SNAPSHOTS"`
	MaxWALs                  int    `yaml:"max_wals"                      env:"ETCD_MAX_WALS"`
	Name                     string `yaml:"name"                          env:"ETCD_NAME"`
	PeerCAFile               string `yaml:"peer_ca_file"                  env:"ETCD_PEER_CA_FILE"`
	PeerCertFile             string `yaml:"peer_cert_file"                env:"ETCD_PEER_CERT_FILE"`
	PeerKeyFile              string `yaml:"peer_key_file"                 env:"ETCD_PEER_KEY_FILE"`
	Proxy                    string `yaml:"proxy"                         env:"ETCD_PROXY" valid:"^(on|off|readonly)$"`
	SnapshotCount            int    `yaml:"snapshot_count"                env:"ETCD_SNAPSHOTCOUNT"`
}
<|endoftext|>


<commit_before>package data

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/golang/glog"
)

const NetworkTimerFreq = time.Second * 60

type Network struct {
	Nodes map[string]*Node
	Timer map[string]time.Time
}

func NewNetwork() *Network {
	n := &Network{
		Nodes: make(map[string]*Node),
		Timer: make(map[string]time.Time)}
	n.startTimer(context.Background())
	return n
}

func (n *Network) startTimer(ctx context.Context) {
	ticker := time.NewTicker(NetworkTimerFreq)
	go func() {
		for {
			select {
			case <-ticker.C:
				for id, insertTime := range n.Timer {
					if time.Since(insertTime) > NetworkTimerFreq {
						delete(n.Timer, id)
						delete(n.Nodes, id)
					}
				}
			case <-ctx.Done():
				glog.Errorf("Monitor Network Timer Done")
				return
			}
		}
	}()
}

func (n *Network) SetNode(node *Node) {
	n.Nodes[node.ID] = node
	n.Timer[node.ID] = time.Now()
}

func (n *Network) ToD3Json() interface{} {
	nodes := make([]map[string]interface{}, 0)
	links := make([]map[string]string, 0)
	streams := make(map[string]map[string]interface{})

	for _, n := range n.Nodes {
		// glog.Infof("Node: %v", n.ID)
		//Add nodes
		nmap := map[string]interface{}{"id": n.ID, "bootnode": n.IsBootNode}
		nodes = append(nodes, nmap)

		//Add links
		for _, conn := range n.Conns {
			links = append(links, map[string]string{"source": conn.N1, "target": conn.N2})
		}

		//Add streams, broadcasters, relayers and subscribers
		checkStrm := func(id string, streams map[string]map[string]interface{}) {
			strm, ok := streams[id]
			if !ok {
				// glog.Infof("New Stream: %v", id)
				strm = map[string]interface{}{"broadcaster": "", "relayers": make([]string, 0), "subscribers": make([]map[string]string, 0)}
				streams[id] = strm
			}
		}

		for _, b := range n.Broadcasts {
			checkStrm(b.StrmID, streams)
			streams[b.StrmID]["broadcaster"] = n.ID
		}
		for _, sub := range n.Subs {
			// glog.Infof("n: %v, strmID: %v", n.ID, sub.StrmID)
			checkStrm(sub.StrmID, streams)
			streams[sub.StrmID]["subscribers"] = append(streams[sub.StrmID]["subscribers"].([]map[string]string), map[string]string{"id": n.ID, "buffer": string(sub.BufferCount)})
		}
		for _, r := range n.Relays {
			// glog.Infof("n: %v, strmID: %v", n.ID, r.StrmID)
			checkStrm(r.StrmID, streams)
			streams[r.StrmID]["relayers"] = append(streams[r.StrmID]["relayers"].([]string), n.ID)
		}

	}

	result := map[string]interface{}{
		"nodes":   nodes,
		"links":   links,
		"streams": streams,
	}

	b, err := json.Marshal(result)
	if err != nil {
		glog.Errorf("Error marshaling json: %v", err)
	}

	var res interface{}
	json.Unmarshal(b, &res)
	return res
}

type Node struct {
	ID                  string                   `json:"id"`
	IsBootNode          bool                     `json:"isBootNode"`
	Conns               []Conn                   `json:"conns"`
	Strms               map[string]Stream        `json:"strms"`
	Broadcasts          map[string]Broadcast     `json:"broadcasts"`
	Relays              map[string]Relay         `json:"relays"`
	Subs                map[string]*Subscription `json:"subs"`
	ConnsInLastHr       uint                     `json:"connsInLastHr"`
	StrmsInLastHr       uint                     `json:"strmsInLastHr"`
	TotalVidLenInLastHr uint                     `json:"TotalVidLenInLastHr"`
	TotalRelayInLastHr  uint                     `json:"TotalRelayInLastHr"`
	AvgCPU              float32                  `json:"avgCPU"`
	AvgMem              uint32                   `json:"avgMem"`
	AvgBandwidth        uint32                   `json:"avgBandwidth"`
	ConnWorker          map[Conn]time.Time       `json:"-"`
}

func NewNode(id string) *Node {
	n := &Node{
		ID:         id,
		Conns:      make([]Conn, 0),
		Strms:      make(map[string]Stream),
		Broadcasts: make(map[string]Broadcast),
		Relays:     make(map[string]Relay),
		Subs:       make(map[string]*Subscription),
		ConnWorker: make(map[Conn]time.Time),
	}

	return n
}

func (n *Node) AddConn(local, remote string) {
	newConn := NewConn(local, remote)
	n.Conns = append(n.Conns, newConn)
	n.ConnsInLastHr++
	n.ConnWorker[newConn] = time.Now()
}

func (n *Node) RemoveConn(local, remote string) {
	rmc := NewConn(local, remote)
	for i, c := range n.Conns {
		if c == rmc {
			n.Conns = append(n.Conns[:i], n.Conns[i+1:]...)
		}
	}
}

func (n *Node) SetBootNode() {
	n.IsBootNode = true
}

func (n *Node) SetStream(id string, size, avgChunkSize uint) {

	// n.Strms = append(n.Strms, Stream{ID: id, Chunks: size, AvgChunkSize: avgChunkSize})
	n.Strms[id] = Stream{ID: id, Chunks: size, AvgChunkSize: avgChunkSize}
}

func (n *Node) RemoveStream(id string) {
	delete(n.Strms, id)
}

func (n *Node) SetBroadcast(strmID string) {
	// n.Broadcasts = append(n.Broadcasts, Broadcast{StrmID: id})
	n.Broadcasts[strmID] = Broadcast{StrmID: strmID}
}

func (n *Node) RemoveBroadcast(strmID string) {
	delete(n.Broadcasts, strmID)
}

func (n *Node) SetSub(strmID string) {
	// n.Subs = append(n.Subs, Subscription{StrmID: strmID})
	n.Subs[strmID] = &Subscription{StrmID: strmID}
}

func (n *Node) RemoveSub(strmID string) {
	delete(n.Subs, strmID)
}

func (n *Node) AddBufferEvent(strmID string) {
	// for i, sub := range n.Subs {
	// 	if sub.StrmID == strmID {
	// 		n.Subs[i].BufferCount++
	// 	}
	// }

	glog.Info("Logging buffer event")
	n.Subs[strmID].BufferCount = n.Subs[strmID].BufferCount + 1
}

func (n *Node) SetRelay(strmID string, remote string) {
	// n.Relays = append(n.Relays, Relay{StrmID: strmID, RemoteN: remote})
	n.Relays[strmID] = Relay{StrmID: strmID, RemoteN: remote}
}

func (n *Node) RemoveRelay(strmID string) {
	delete(n.Relays, strmID)
}

func (n *Node) SubmitToCollector(endpoint string) {
	if endpoint == "" {
		return
	}

	enc, err := json.Marshal(n)
	if err != nil {
		glog.Errorf("Error marshaling node status: %v", err)
		return
	}

	req, err := http.NewRequest("POST", endpoint, bytes.NewBuffer(enc))
	if err != nil {
		glog.Errorf("Error creating new request: %v", err)
		return
	}
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("Couldn't connect to the event server", err)
		return
	}
	defer resp.Body.Close()
}

type Conn struct {
	N1 string `json:"n1"`
	N2 string `json:"n2"`
}

func NewConn(local, remote string) Conn {
	if strings.Compare(local, remote) > 0 {
		return Conn{N1: local, N2: remote}
	} else {
		return Conn{N1: remote, N2: local}
	}
}

type Stream struct {
	ID           string `json:"id"`
	Chunks       uint   `json:"chunks"`
	AvgChunkSize uint   `json:"avgChunkSize"`
}

type Broadcast struct {
	StrmID string
}

type Relay struct {
	StrmID  string
	RemoteN string
}

type Subscription struct {
	StrmID      string
	BufferCount uint
}
<commit_msg>change json<commit_after>package data

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/golang/glog"
)

const NetworkTimerFreq = time.Second * 60

type Network struct {
	Nodes map[string]*Node
	Timer map[string]time.Time
}

func NewNetwork() *Network {
	n := &Network{
		Nodes: make(map[string]*Node),
		Timer: make(map[string]time.Time)}
	n.startTimer(context.Background())
	return n
}

func (n *Network) startTimer(ctx context.Context) {
	ticker := time.NewTicker(NetworkTimerFreq)
	go func() {
		for {
			select {
			case <-ticker.C:
				for id, insertTime := range n.Timer {
					if time.Since(insertTime) > NetworkTimerFreq {
						delete(n.Timer, id)
						delete(n.Nodes, id)
					}
				}
			case <-ctx.Done():
				glog.Errorf("Monitor Network Timer Done")
				return
			}
		}
	}()
}

func (n *Network) SetNode(node *Node) {
	n.Nodes[node.ID] = node
	n.Timer[node.ID] = time.Now()
}

func (n *Network) ToD3Json() interface{} {
	nodes := make([]map[string]interface{}, 0)
	links := make([]map[string]string, 0)
	streams := make(map[string]map[string]interface{})

	for _, n := range n.Nodes {
		// glog.Infof("Node: %v", n.ID)
		//Add nodes
		nmap := map[string]interface{}{"id": n.ID, "bootnode": n.IsBootNode}
		nodes = append(nodes, nmap)

		//Add links
		for _, conn := range n.Conns {
			links = append(links, map[string]string{"source": conn.N1, "target": conn.N2})
		}

		//Add streams, broadcasters, relayers and subscribers
		checkStrm := func(id string, streams map[string]map[string]interface{}) {
			strm, ok := streams[id]
			if !ok {
				// glog.Infof("New Stream: %v", id)
				strm = map[string]interface{}{"broadcaster": "", "relayers": make([]string, 0), "subscribers": make([]map[string]string, 0)}
				streams[id] = strm
			}
		}

		for _, b := range n.Broadcasts {
			checkStrm(b.StrmID, streams)
			streams[b.StrmID]["broadcaster"] = n.ID
		}
		for _, sub := range n.Subs {
			// glog.Infof("n: %v, strmID: %v", n.ID, sub.StrmID)
			checkStrm(sub.StrmID, streams)
			streams[sub.StrmID]["subscribers"] = append(streams[sub.StrmID]["subscribers"].([]map[string]string), map[string]string{"id": n.ID, "buffer": string(sub.BufferCount)})
		}
		for _, r := range n.Relays {
			// glog.Infof("n: %v, strmID: %v", n.ID, r.StrmID)
			checkStrm(r.StrmID, streams)
			streams[r.StrmID]["relayers"] = append(streams[r.StrmID]["relayers"].([]string), n.ID)
		}

	}

	result := map[string]interface{}{
		"nodes":   nodes,
		"links":   links,
		"streams": streams,
	}

	b, err := json.Marshal(result)
	if err != nil {
		glog.Errorf("Error marshaling json: %v", err)
	}

	var res interface{}
	json.Unmarshal(b, &res)
	return res
}

type Node struct {
	ID                  string                   `json:"id"`
	IsBootNode          bool                     `json:"isBootNode"`
	Conns               []Conn                   `json:"conns"`
	Strms               map[string]Stream        `json:"strms"`
	Broadcasts          map[string]Broadcast     `json:"broadcasts"`
	Relays              map[string]Relay         `json:"relays"`
	Subs                map[string]*Subscription `json:"subs"`
	ConnsInLastHr       uint                     `json:"connsInLastHr"`
	StrmsInLastHr       uint                     `json:"strmsInLastHr"`
	TotalVidLenInLastHr uint                     `json:"totalVidLenInLastHr"`
	TotalRelayInLastHr  uint                     `json:"totalRelayInLastHr"`
	AvgCPU              float32                  `json:"avgCPU"`
	AvgMem              uint32                   `json:"avgMem"`
	AvgBandwidth        uint32                   `json:"avgBandwidth"`
	ConnWorker          map[Conn]time.Time       `json:"-"`
}

func NewNode(id string) *Node {
	n := &Node{
		ID:         id,
		Conns:      make([]Conn, 0),
		Strms:      make(map[string]Stream),
		Broadcasts: make(map[string]Broadcast),
		Relays:     make(map[string]Relay),
		Subs:       make(map[string]*Subscription),
		ConnWorker: make(map[Conn]time.Time),
	}

	return n
}

func (n *Node) AddConn(local, remote string) {
	newConn := NewConn(local, remote)
	n.Conns = append(n.Conns, newConn)
	n.ConnsInLastHr++
	n.ConnWorker[newConn] = time.Now()
}

func (n *Node) RemoveConn(local, remote string) {
	rmc := NewConn(local, remote)
	for i, c := range n.Conns {
		if c == rmc {
			n.Conns = append(n.Conns[:i], n.Conns[i+1:]...)
		}
	}
}

func (n *Node) SetBootNode() {
	n.IsBootNode = true
}

func (n *Node) SetStream(id string, size, avgChunkSize uint) {

	// n.Strms = append(n.Strms, Stream{ID: id, Chunks: size, AvgChunkSize: avgChunkSize})
	n.Strms[id] = Stream{ID: id, Chunks: size, AvgChunkSize: avgChunkSize}
}

func (n *Node) RemoveStream(id string) {
	delete(n.Strms, id)
}

func (n *Node) SetBroadcast(strmID string) {
	// n.Broadcasts = append(n.Broadcasts, Broadcast{StrmID: id})
	n.Broadcasts[strmID] = Broadcast{StrmID: strmID}
}

func (n *Node) RemoveBroadcast(strmID string) {
	delete(n.Broadcasts, strmID)
}

func (n *Node) SetSub(strmID string) {
	// n.Subs = append(n.Subs, Subscription{StrmID: strmID})
	n.Subs[strmID] = &Subscription{StrmID: strmID}
}

func (n *Node) RemoveSub(strmID string) {
	delete(n.Subs, strmID)
}

func (n *Node) AddBufferEvent(strmID string) {
	// for i, sub := range n.Subs {
	// 	if sub.StrmID == strmID {
	// 		n.Subs[i].BufferCount++
	// 	}
	// }

	glog.Info("Logging buffer event")
	n.Subs[strmID].BufferCount = n.Subs[strmID].BufferCount + 1
}

func (n *Node) SetRelay(strmID string, remote string) {
	// n.Relays = append(n.Relays, Relay{StrmID: strmID, RemoteN: remote})
	n.Relays[strmID] = Relay{StrmID: strmID, RemoteN: remote}
}

func (n *Node) RemoveRelay(strmID string) {
	delete(n.Relays, strmID)
}

func (n *Node) SubmitToCollector(endpoint string) {
	if endpoint == "" {
		return
	}

	enc, err := json.Marshal(n)
	if err != nil {
		glog.Errorf("Error marshaling node status: %v", err)
		return
	}

	req, err := http.NewRequest("POST", endpoint, bytes.NewBuffer(enc))
	if err != nil {
		glog.Errorf("Error creating new request: %v", err)
		return
	}
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("Couldn't connect to the event server", err)
		return
	}
	defer resp.Body.Close()
}

type Conn struct {
	N1 string `json:"n1"`
	N2 string `json:"n2"`
}

func NewConn(local, remote string) Conn {
	if strings.Compare(local, remote) > 0 {
		return Conn{N1: local, N2: remote}
	} else {
		return Conn{N1: remote, N2: local}
	}
}

type Stream struct {
	ID           string `json:"id"`
	Chunks       uint   `json:"chunks"`
	AvgChunkSize uint   `json:"avgChunkSize"`
}

type Broadcast struct {
	StrmID string
}

type Relay struct {
	StrmID  string
	RemoteN string
}

type Subscription struct {
	StrmID      string
	BufferCount uint
}
<|endoftext|>


<commit_before>// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package awsecr

import (
	"errors"
	"net/http"
	"regexp"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	awsecrapi "github.com/aws/aws-sdk-go/service/ecr"
	"github.com/goharbor/harbor/src/common/utils/log"
	"github.com/goharbor/harbor/src/common/utils/registry"
	adp "github.com/goharbor/harbor/src/replication/adapter"
	"github.com/goharbor/harbor/src/replication/adapter/native"
	"github.com/goharbor/harbor/src/replication/model"
)

const (
	regionPattern = "https://(?:api|\\d+\\.dkr)\\.ecr\\.([\\w\\-]+)\\.amazonaws\\.com"
)

var (
	regionRegexp = regexp.MustCompile(regionPattern)
)

func init() {
	if err := adp.RegisterFactory(model.RegistryTypeAwsEcr, new(factory)); err != nil {
		log.Errorf("failed to register factory for %s: %v", model.RegistryTypeAwsEcr, err)
		return
	}
	log.Infof("the factory for adapter %s registered", model.RegistryTypeAwsEcr)
}

func newAdapter(registry *model.Registry) (*adapter, error) {
	region, err := parseRegion(registry.URL)
	if err != nil {
		return nil, err
	}
	authorizer := NewAuth(region, registry.Credential.AccessKey, registry.Credential.AccessSecret, registry.Insecure)
	dockerRegistry, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer)
	if err != nil {
		return nil, err
	}
	return &adapter{
		registry: registry,
		Adapter:  dockerRegistry,
		region:   region,
	}, nil
}

func parseRegion(url string) (string, error) {
	rs := regionRegexp.FindStringSubmatch(url)
	if rs == nil {
		return "", errors.New("Bad aws url")
	}
	return rs[1], nil
}

type factory struct {
}

// Create ...
func (f *factory) Create(r *model.Registry) (adp.Adapter, error) {
	return newAdapter(r)
}

// AdapterPattern ...
func (f *factory) AdapterPattern() *model.AdapterPattern {
	return getAdapterInfo()
}

type adapter struct {
	*native.Adapter
	registry      *model.Registry
	region        string
	forceEndpoint *string
}

func (*adapter) Info() (info *model.RegistryInfo, err error) {
	return &model.RegistryInfo{
		Type: model.RegistryTypeAwsEcr,
		SupportedResourceTypes: []model.ResourceType{
			model.ResourceTypeImage,
		},
		SupportedResourceFilters: []*model.FilterStyle{
			{
				Type:  model.FilterTypeName,
				Style: model.FilterStyleTypeText,
			},
			{
				Type:  model.FilterTypeTag,
				Style: model.FilterStyleTypeText,
			},
		},
		SupportedTriggers: []model.TriggerType{
			model.TriggerTypeManual,
			model.TriggerTypeScheduled,
		},
	}, nil
}

func getAdapterInfo() *model.AdapterPattern {
	info := &model.AdapterPattern{
		EndpointPattern: &model.EndpointPattern{
			EndpointType: model.EndpointPatternTypeList,
			Endpoints: []*model.Endpoint{
				{
					Key:   "ap-northeast-1",
					Value: "https://api.ecr.ap-northeast-1.amazonaws.com",
				},
				{
					Key:   "us-east-1",
					Value: "https://api.ecr.us-east-1.amazonaws.com",
				},
				{
					Key:   "us-east-2",
					Value: "https://api.ecr.us-east-2.amazonaws.com",
				},
				{
					Key:   "us-west-1",
					Value: "https://api.ecr.us-west-1.amazonaws.com",
				},
				{
					Key:   "us-west-2",
					Value: "https://api.ecr.us-west-2.amazonaws.com",
				},
				{
					Key:   "ap-east-1",
					Value: "https://api.ecr.ap-east-1.amazonaws.com",
				},
				{
					Key:   "ap-south-1",
					Value: "https://api.ecr.ap-south-1.amazonaws.com",
				},
				{
					Key:   "ap-northeast-2",
					Value: "https://api.ecr.ap-northeast-2.amazonaws.com",
				},
				{
					Key:   "ap-southeast-1",
					Value: "https://api.ecr.ap-southeast-1.amazonaws.com",
				},
				{
					Key:   "ap-southeast-2",
					Value: "https://api.ecr.ap-southeast-2.amazonaws.com",
				},
				{
					Key:   "ca-central-1",
					Value: "https://api.ecr.ca-central-1.amazonaws.com",
				},
				{
					Key:   "eu-central-1",
					Value: "https://api.ecr.eu-central-1.amazonaws.com",
				},
				{
					Key:   "eu-west-1",
					Value: "https://api.ecr.eu-west-1.amazonaws.com",
				},
				{
					Key:   "eu-west-2",
					Value: "https://api.ecr.eu-west-2.amazonaws.com",
				},
				{
					Key:   "eu-west-3",
					Value: "https://api.ecr.eu-west-3.amazonaws.com",
				},
				{
					Key:   "eu-north-1",
					Value: "https://api.ecr.eu-north-1.amazonaws.com",
				},
				{
					Key:   "sa-east-1",
					Value: "https://api.ecr.sa-east-1.amazonaws.com",
				},
			},
		},
	}
	return info
}

// HealthCheck checks health status of a registry
func (a *adapter) HealthCheck() (model.HealthStatus, error) {
	if a.registry.Credential == nil ||
		len(a.registry.Credential.AccessKey) == 0 || len(a.registry.Credential.AccessSecret) == 0 {
		log.Errorf("no credential to ping registry %s", a.registry.URL)
		return model.Unhealthy, nil
	}
	if err := a.PingGet(); err != nil {
		log.Errorf("failed to ping registry %s: %v", a.registry.URL, err)
		return model.Unhealthy, nil
	}
	return model.Healthy, nil
}

// PrepareForPush nothing need to do.
func (a *adapter) PrepareForPush(resources []*model.Resource) error {
	for _, resource := range resources {
		if resource == nil {
			return errors.New("the resource cannot be nil")
		}
		if resource.Metadata == nil {
			return errors.New("the metadata of resource cannot be nil")
		}
		if resource.Metadata.Repository == nil {
			return errors.New("the namespace of resource cannot be nil")
		}
		if len(resource.Metadata.Repository.Name) == 0 {
			return errors.New("the name of the namespace cannot be nil")
		}

		err := a.createRepository(resource.Metadata.Repository.Name)
		if err != nil {
			return err
		}
	}
	return nil
}

func (a *adapter) createRepository(repository string) error {
	if a.registry.Credential == nil ||
		len(a.registry.Credential.AccessKey) == 0 || len(a.registry.Credential.AccessSecret) == 0 {
		return errors.New("no credential ")
	}
	cred := credentials.NewStaticCredentials(
		a.registry.Credential.AccessKey,
		a.registry.Credential.AccessSecret,
		"")
	if a.region == "" {
		return errors.New("no region parsed")
	}
	config := &aws.Config{
		Credentials: cred,
		Region:      &a.region,
		HTTPClient: &http.Client{
			Transport: registry.GetHTTPTransport(a.registry.Insecure),
		},
	}
	if a.forceEndpoint != nil {
		config.Endpoint = a.forceEndpoint
	}
	sess := session.Must(session.NewSession(config))

	svc := awsecrapi.New(sess)

	_, err := svc.CreateRepository(&awsecrapi.CreateRepositoryInput{
		RepositoryName: &repository,
	})
	if err != nil {
		if e, ok := err.(awserr.Error); ok {
			if e.Code() == awsecrapi.ErrCodeRepositoryAlreadyExistsException {
				return nil
			}
		}
		return err
	}
	return nil
}
<commit_msg>fix(replication): aws ecr delete image<commit_after>// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package awsecr

import (
	"errors"
	"net/http"
	"regexp"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	awsecrapi "github.com/aws/aws-sdk-go/service/ecr"
	"github.com/goharbor/harbor/src/common/utils/log"
	"github.com/goharbor/harbor/src/common/utils/registry"
	adp "github.com/goharbor/harbor/src/replication/adapter"
	"github.com/goharbor/harbor/src/replication/adapter/native"
	"github.com/goharbor/harbor/src/replication/model"
)

const (
	regionPattern = "https://(?:api|\\d+\\.dkr)\\.ecr\\.([\\w\\-]+)\\.amazonaws\\.com"
)

var (
	regionRegexp = regexp.MustCompile(regionPattern)
)

func init() {
	if err := adp.RegisterFactory(model.RegistryTypeAwsEcr, new(factory)); err != nil {
		log.Errorf("failed to register factory for %s: %v", model.RegistryTypeAwsEcr, err)
		return
	}
	log.Infof("the factory for adapter %s registered", model.RegistryTypeAwsEcr)
}

func newAdapter(registry *model.Registry) (*adapter, error) {
	region, err := parseRegion(registry.URL)
	if err != nil {
		return nil, err
	}
	authorizer := NewAuth(region, registry.Credential.AccessKey, registry.Credential.AccessSecret, registry.Insecure)
	dockerRegistry, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer)
	if err != nil {
		return nil, err
	}
	return &adapter{
		registry: registry,
		Adapter:  dockerRegistry,
		region:   region,
	}, nil
}

func parseRegion(url string) (string, error) {
	rs := regionRegexp.FindStringSubmatch(url)
	if rs == nil {
		return "", errors.New("Bad aws url")
	}
	return rs[1], nil
}

type factory struct {
}

// Create ...
func (f *factory) Create(r *model.Registry) (adp.Adapter, error) {
	return newAdapter(r)
}

// AdapterPattern ...
func (f *factory) AdapterPattern() *model.AdapterPattern {
	return getAdapterInfo()
}

type adapter struct {
	*native.Adapter
	registry      *model.Registry
	region        string
	forceEndpoint *string
}

func (*adapter) Info() (info *model.RegistryInfo, err error) {
	return &model.RegistryInfo{
		Type: model.RegistryTypeAwsEcr,
		SupportedResourceTypes: []model.ResourceType{
			model.ResourceTypeImage,
		},
		SupportedResourceFilters: []*model.FilterStyle{
			{
				Type:  model.FilterTypeName,
				Style: model.FilterStyleTypeText,
			},
			{
				Type:  model.FilterTypeTag,
				Style: model.FilterStyleTypeText,
			},
		},
		SupportedTriggers: []model.TriggerType{
			model.TriggerTypeManual,
			model.TriggerTypeScheduled,
		},
	}, nil
}

func getAdapterInfo() *model.AdapterPattern {
	info := &model.AdapterPattern{
		EndpointPattern: &model.EndpointPattern{
			EndpointType: model.EndpointPatternTypeList,
			Endpoints: []*model.Endpoint{
				{
					Key:   "ap-northeast-1",
					Value: "https://api.ecr.ap-northeast-1.amazonaws.com",
				},
				{
					Key:   "us-east-1",
					Value: "https://api.ecr.us-east-1.amazonaws.com",
				},
				{
					Key:   "us-east-2",
					Value: "https://api.ecr.us-east-2.amazonaws.com",
				},
				{
					Key:   "us-west-1",
					Value: "https://api.ecr.us-west-1.amazonaws.com",
				},
				{
					Key:   "us-west-2",
					Value: "https://api.ecr.us-west-2.amazonaws.com",
				},
				{
					Key:   "ap-east-1",
					Value: "https://api.ecr.ap-east-1.amazonaws.com",
				},
				{
					Key:   "ap-south-1",
					Value: "https://api.ecr.ap-south-1.amazonaws.com",
				},
				{
					Key:   "ap-northeast-2",
					Value: "https://api.ecr.ap-northeast-2.amazonaws.com",
				},
				{
					Key:   "ap-southeast-1",
					Value: "https://api.ecr.ap-southeast-1.amazonaws.com",
				},
				{
					Key:   "ap-southeast-2",
					Value: "https://api.ecr.ap-southeast-2.amazonaws.com",
				},
				{
					Key:   "ca-central-1",
					Value: "https://api.ecr.ca-central-1.amazonaws.com",
				},
				{
					Key:   "eu-central-1",
					Value: "https://api.ecr.eu-central-1.amazonaws.com",
				},
				{
					Key:   "eu-west-1",
					Value: "https://api.ecr.eu-west-1.amazonaws.com",
				},
				{
					Key:   "eu-west-2",
					Value: "https://api.ecr.eu-west-2.amazonaws.com",
				},
				{
					Key:   "eu-west-3",
					Value: "https://api.ecr.eu-west-3.amazonaws.com",
				},
				{
					Key:   "eu-north-1",
					Value: "https://api.ecr.eu-north-1.amazonaws.com",
				},
				{
					Key:   "sa-east-1",
					Value: "https://api.ecr.sa-east-1.amazonaws.com",
				},
			},
		},
	}
	return info
}

// HealthCheck checks health status of a registry
func (a *adapter) HealthCheck() (model.HealthStatus, error) {
	if a.registry.Credential == nil ||
		len(a.registry.Credential.AccessKey) == 0 || len(a.registry.Credential.AccessSecret) == 0 {
		log.Errorf("no credential to ping registry %s", a.registry.URL)
		return model.Unhealthy, nil
	}
	if err := a.PingGet(); err != nil {
		log.Errorf("failed to ping registry %s: %v", a.registry.URL, err)
		return model.Unhealthy, nil
	}
	return model.Healthy, nil
}

// PrepareForPush nothing need to do.
func (a *adapter) PrepareForPush(resources []*model.Resource) error {
	for _, resource := range resources {
		if resource == nil {
			return errors.New("the resource cannot be nil")
		}
		if resource.Metadata == nil {
			return errors.New("the metadata of resource cannot be nil")
		}
		if resource.Metadata.Repository == nil {
			return errors.New("the namespace of resource cannot be nil")
		}
		if len(resource.Metadata.Repository.Name) == 0 {
			return errors.New("the name of the namespace cannot be nil")
		}

		err := a.createRepository(resource.Metadata.Repository.Name)
		if err != nil {
			return err
		}
	}
	return nil
}

func (a *adapter) createRepository(repository string) error {
	if a.registry.Credential == nil ||
		len(a.registry.Credential.AccessKey) == 0 || len(a.registry.Credential.AccessSecret) == 0 {
		return errors.New("no credential ")
	}
	cred := credentials.NewStaticCredentials(
		a.registry.Credential.AccessKey,
		a.registry.Credential.AccessSecret,
		"")
	if a.region == "" {
		return errors.New("no region parsed")
	}
	config := &aws.Config{
		Credentials: cred,
		Region:      &a.region,
		HTTPClient: &http.Client{
			Transport: registry.GetHTTPTransport(a.registry.Insecure),
		},
	}
	if a.forceEndpoint != nil {
		config.Endpoint = a.forceEndpoint
	}
	sess := session.Must(session.NewSession(config))

	svc := awsecrapi.New(sess)

	_, err := svc.CreateRepository(&awsecrapi.CreateRepositoryInput{
		RepositoryName: &repository,
	})
	if err != nil {
		if e, ok := err.(awserr.Error); ok {
			if e.Code() == awsecrapi.ErrCodeRepositoryAlreadyExistsException {
				return nil
			}
		}
		return err
	}
	return nil
}

// DeleteManifest ...
func (a *adapter) DeleteManifest(repository, reference string) error {
	// AWS doesn't implement standard OCI delete manifest API, so use it's sdk.
	if a.registry.Credential == nil ||
		len(a.registry.Credential.AccessKey) == 0 || len(a.registry.Credential.AccessSecret) == 0 {
		return errors.New("no credential ")
	}
	cred := credentials.NewStaticCredentials(
		a.registry.Credential.AccessKey,
		a.registry.Credential.AccessSecret,
		"")
	if a.region == "" {
		return errors.New("no region parsed")
	}
	config := &aws.Config{
		Credentials: cred,
		Region:      &a.region,
		HTTPClient: &http.Client{
			Transport: registry.GetHTTPTransport(a.registry.Insecure),
		},
	}
	if a.forceEndpoint != nil {
		config.Endpoint = a.forceEndpoint
	}
	sess := session.Must(session.NewSession(config))

	svc := awsecrapi.New(sess)

	_, err := svc.BatchDeleteImage(&awsecrapi.BatchDeleteImageInput{
		RepositoryName: &repository,
		ImageIds:       []*awsecrapi.ImageIdentifier{{ImageTag: &reference}},
	})
	return err
}
<|endoftext|>


<commit_before>package suggest

import (
	"fmt"

	"github.com/remeh/smartwitter/storage"
	"github.com/remeh/uuid"

	"github.com/lib/pq"
)

// SetKeywords stores the given keywords for the user.
// It also resets the last run on these, forcing a near-to-be executed
// crawling.
func SetKeywords(userUid uuid.UUID, keywords []string, position int) error {
	if position < 0 {
		return fmt.Errorf("SetKeywords: position <1")
	}

	if _, err := storage.DB().Exec(`
		UPDATE "twitter_keywords_watcher"
		SET
			"keywords" = $1
		WHERE
			"user_uid" = $2
			AND
			"position" = $3
	`, pq.Array(keywords), userUid, position); err != nil {
		return err
	}

	return nil
}
<commit_msg>suggest: load user keywords.<commit_after>package suggest

import (
	"fmt"
	"time"

	"github.com/remeh/smartwitter/log"
	"github.com/remeh/smartwitter/storage"
	"github.com/remeh/uuid"

	"github.com/lib/pq"
)

type Keywords []keywords
type keywords struct {
	Uid          uuid.UUID `json:"-"`
	UserUid      uuid.UUID `json:"-"`
	Keywords     []string  `json:"keywords"`
	Position     int       `json:"position"`
	CreationTime time.Time `json:"creation_time"`
	LastUpdate   time.Time `json:"last_update"`
}

func (k Keywords) Len() int               { return len(k) }
func (k Keywords) Less(i int, j int) bool { return k[i].Position < k[j].Position }
func (k Keywords) Swap(i int, j int)      { k[i], k[j] = k[j], k[j] }

// SetKeywords stores the given keywords for the user.
// It also resets the last run on these, forcing a near-to-be executed
// crawling.
func SetKeywords(userUid uuid.UUID, keywords []string, position int) error {
	if position < 0 {
		return fmt.Errorf("SetKeywords: position <1")
	}

	if _, err := storage.DB().Exec(`
		UPDATE "twitter_keywords_watcher"
		SET
			"keywords" = $1
		WHERE
			"user_uid" = $2
			AND
			"position" = $3
	`, pq.Array(keywords), userUid, position); err != nil {
		return err
	}

	return nil
}

// GetKeywords returns users watched keywords.
func GetKeywords(userUid uuid.UUID) (Keywords, error) {
	rv := make(Keywords, 0)

	if userUid.IsNil() {
		log.Warning("GetKeywords called with a nil userUid")
		return rv, nil
	}

	rows, err := storage.DB().Query(`
		SELECT "uid", "keywords", "position", "creation_time", "last_update"
		FROM
			"twitter_keywords_watcher"
		WHERE
			"user_uid" = $1
		ORDER BY "position"
	`, userUid)
	if err != nil {
		return rv, log.Err("GetKeywords", err)
	}

	defer rows.Close()

	for rows.Next() {
		k := keywords{}
		if err := rows.Scan(
			&k.Uid,
			pq.Array(&k.Keywords),
			&k.Position,
			&k.CreationTime,
			&k.LastUpdate,
		); err != nil {
			return rv, log.Err("GetKeywords", err)
		}
		k.UserUid = userUid
		rv = append(rv, k)
	}
	return rv, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package ec2

import (
	"github.com/juju/errors"
	"launchpad.net/goamz/ec2"

	"github.com/juju/juju/environs"
	"github.com/juju/juju/storage"
)

const (
	ebsStorageSource = "ebs"

	// minRootDiskSize is the minimum/default size (in mebibytes) for ec2 root disks.
	minRootDiskSize uint64 = 8 * 1024

	// volumeSizeMax is the maximum disk size (in mebibytes) for EBS volumes.
	volumeSizeMax = 1024 * 1024 // 1024 GiB
)

// getBlockDeviceMappings translates a StartInstanceParams into
// BlockDeviceMappings.
//
// The first entry is always the root disk mapping, instance stores
// (ephemeral disks) last.
func getBlockDeviceMappings(
	virtType string,
	args *environs.StartInstanceParams,
) (
	[]ec2.BlockDeviceMapping, []storage.BlockDevice, error,
) {
	rootDiskSize := minRootDiskSize
	if args.Constraints.RootDisk != nil {
		if *args.Constraints.RootDisk >= minRootDiskSize {
			rootDiskSize = *args.Constraints.RootDisk
		} else {
			logger.Infof(
				"Ignoring root-disk constraint of %dM because it is smaller than the EC2 image size of %dM",
				*args.Constraints.RootDisk,
				minRootDiskSize,
			)
		}
	}

	// The first block device is for the root disk.
	blockDeviceMappings := []ec2.BlockDeviceMapping{{
		DeviceName: "/dev/sda1",
		VolumeSize: int64(mibToGib(rootDiskSize)),
	}}

	// Not all machines have this many instance stores.
	// Instances will be started with as many of the
	// instance stores as they can support.
	blockDeviceMappings = append(blockDeviceMappings, []ec2.BlockDeviceMapping{{
		VirtualName: "ephemeral0",
		DeviceName:  "/dev/sdb",
	}, {
		VirtualName: "ephemeral1",
		DeviceName:  "/dev/sdc",
	}, {
		VirtualName: "ephemeral2",
		DeviceName:  "/dev/sdd",
	}, {
		VirtualName: "ephemeral3",
		DeviceName:  "/dev/sde",
	}}...)

	// TODO(axw) if preference is to use ephemeral, use ephemeral
	// until the instance stores run out. We'll need to know how
	// many there are and how big each one is. We also need to
	// unmap ephemeral0 in cloud-init.

	disks := make([]storage.BlockDevice, len(args.Disks))
	mappings := make([]ec2.BlockDeviceMapping, len(args.Disks))
	nextDeviceName := blockDeviceNamer(virtType == paravirtual)
	for i, params := range args.Disks {
		// Check minimum constraints can be satisfied.
		if err := validateDiskParams(params); err != nil {
			return nil, nil, errors.Annotate(err, "invalid disk parameters")
		}
		requestDeviceName, actualDeviceName, err := nextDeviceName()
		if err != nil {
			// Can't allocate any more disks.
			return nil, nil, err
		}
		mapping := ec2.BlockDeviceMapping{
			VolumeSize: int64(mibToGib(params.Size)),
			DeviceName: requestDeviceName,
			// TODO(axw) VolumeType, IOPS and DeleteOnTermination
		}
		disk := storage.BlockDevice{
			DeviceName: actualDeviceName,
			Size:       gibToMib(uint64(mapping.VolumeSize)),
			// ProviderId will be filled in once the instance has
			// been created, which will create the volumes too.
		}
		mappings[i] = mapping
		disks[i] = disk
	}
	return blockDeviceMappings, disks, nil
}

// validateDiskParams validates the disk parameters.
func validateDiskParams(params storage.DiskParams) error {
	if params.Size > volumeSizeMax {
		return errors.Errorf("%d MiB exceeds the maximum of %d MiB", params.Size, volumeSizeMax)
	}
	return nil
}

// mibToGib converts mebibytes to gibibytes.
// AWS expects GiB, we work in MiB; round up
// to nearest GiB.
func mibToGib(m uint64) uint64 {
	return (m + 1023) / 1024
}

// gibToMib converts gibibytes to mebibytes.
func gibToMib(g uint64) uint64 {
	return g * 1024
}

var errTooManyVolumes = errors.New("too many EBS volumes to attach")

// blockDeviceNamer returns a function that cycles through block device names.
//
// The returned function returns the device name that should be used in
// requests to the EC2 API, and and also the (kernel) device name as it
// will appear on the machine.
//
// See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html
func blockDeviceNamer(numbers bool) func() (requestName, actualName string, err error) {
	const (
		// deviceLetterMin is the first letter to use for EBS block device names.
		deviceLetterMin = 'f'
		// deviceLetterMax is the last letter to use for EBS block device names.
		deviceLetterMax = 'p'
		// deviceNumMax is the maximum value for trailing numbers on block device name.
		deviceNumMax = 6
		// devicePrefix is the prefix for device names specified when creating volumes.
		devicePrefix = "/dev/sd"
		// renamedDevicePrefix is the prefix for device names after they have
		// been renamed. This should replace "devicePrefix" in the device name
		// when recording the block device info in state.
		renamedDevicePrefix = "xvd"
	)
	var n int
	letterRepeats := 1
	if numbers {
		letterRepeats = deviceNumMax
	}
	return func() (string, string, error) {
		letter := deviceLetterMin + (n / letterRepeats)
		if letter > deviceLetterMax {
			return "", "", errTooManyVolumes
		}
		deviceName := devicePrefix + string(letter)
		if numbers {
			deviceName += string('1' + (n % deviceNumMax))
		}
		n++
		realDeviceName := renamedDevicePrefix + deviceName[len(devicePrefix):]
		return deviceName, realDeviceName, nil
	}
}
<commit_msg>provider/ec2: add suffixes to size constants/vars<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package ec2

import (
	"github.com/juju/errors"
	"launchpad.net/goamz/ec2"

	"github.com/juju/juju/environs"
	"github.com/juju/juju/storage"
)

const (
	ebsStorageSource = "ebs"

	// minRootDiskSizeMiB is the minimum/default size (in mebibytes) for ec2 root disks.
	minRootDiskSizeMiB uint64 = 8 * 1024

	// volumeSizeMaxMiB is the maximum disk size (in mebibytes) for EBS volumes.
	volumeSizeMaxMiB = 1024 * 1024 // 1024 GiB
)

// getBlockDeviceMappings translates a StartInstanceParams into
// BlockDeviceMappings.
//
// The first entry is always the root disk mapping, instance stores
// (ephemeral disks) last.
func getBlockDeviceMappings(
	virtType string,
	args *environs.StartInstanceParams,
) (
	[]ec2.BlockDeviceMapping, []storage.BlockDevice, error,
) {
	rootDiskSizeMiB := minRootDiskSizeMiB
	if args.Constraints.RootDisk != nil {
		if *args.Constraints.RootDisk >= minRootDiskSizeMiB {
			rootDiskSizeMiB = *args.Constraints.RootDisk
		} else {
			logger.Infof(
				"Ignoring root-disk constraint of %dM because it is smaller than the EC2 image size of %dM",
				*args.Constraints.RootDisk,
				minRootDiskSizeMiB,
			)
		}
	}

	// The first block device is for the root disk.
	blockDeviceMappings := []ec2.BlockDeviceMapping{{
		DeviceName: "/dev/sda1",
		VolumeSize: int64(mibToGib(rootDiskSizeMiB)),
	}}

	// Not all machines have this many instance stores.
	// Instances will be started with as many of the
	// instance stores as they can support.
	blockDeviceMappings = append(blockDeviceMappings, []ec2.BlockDeviceMapping{{
		VirtualName: "ephemeral0",
		DeviceName:  "/dev/sdb",
	}, {
		VirtualName: "ephemeral1",
		DeviceName:  "/dev/sdc",
	}, {
		VirtualName: "ephemeral2",
		DeviceName:  "/dev/sdd",
	}, {
		VirtualName: "ephemeral3",
		DeviceName:  "/dev/sde",
	}}...)

	// TODO(axw) if preference is to use ephemeral, use ephemeral
	// until the instance stores run out. We'll need to know how
	// many there are and how big each one is. We also need to
	// unmap ephemeral0 in cloud-init.

	disks := make([]storage.BlockDevice, len(args.Disks))
	mappings := make([]ec2.BlockDeviceMapping, len(args.Disks))
	nextDeviceName := blockDeviceNamer(virtType == paravirtual)
	for i, params := range args.Disks {
		// Check minimum constraints can be satisfied.
		if err := validateDiskParams(params); err != nil {
			return nil, nil, errors.Annotate(err, "invalid disk parameters")
		}
		requestDeviceName, actualDeviceName, err := nextDeviceName()
		if err != nil {
			// Can't allocate any more disks.
			return nil, nil, err
		}
		mapping := ec2.BlockDeviceMapping{
			VolumeSize: int64(mibToGib(params.Size)),
			DeviceName: requestDeviceName,
			// TODO(axw) VolumeType, IOPS and DeleteOnTermination
		}
		disk := storage.BlockDevice{
			DeviceName: actualDeviceName,
			Size:       gibToMib(uint64(mapping.VolumeSize)),
			// ProviderId will be filled in once the instance has
			// been created, which will create the volumes too.
		}
		mappings[i] = mapping
		disks[i] = disk
	}
	return blockDeviceMappings, disks, nil
}

// validateDiskParams validates the disk parameters.
func validateDiskParams(params storage.DiskParams) error {
	if params.Size > volumeSizeMaxMiB {
		return errors.Errorf("%d MiB exceeds the maximum of %d MiB", params.Size, volumeSizeMaxMiB)
	}
	return nil
}

// mibToGib converts mebibytes to gibibytes.
// AWS expects GiB, we work in MiB; round up
// to nearest GiB.
func mibToGib(m uint64) uint64 {
	return (m + 1023) / 1024
}

// gibToMib converts gibibytes to mebibytes.
func gibToMib(g uint64) uint64 {
	return g * 1024
}

var errTooManyVolumes = errors.New("too many EBS volumes to attach")

// blockDeviceNamer returns a function that cycles through block device names.
//
// The returned function returns the device name that should be used in
// requests to the EC2 API, and and also the (kernel) device name as it
// will appear on the machine.
//
// See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html
func blockDeviceNamer(numbers bool) func() (requestName, actualName string, err error) {
	const (
		// deviceLetterMin is the first letter to use for EBS block device names.
		deviceLetterMin = 'f'
		// deviceLetterMax is the last letter to use for EBS block device names.
		deviceLetterMax = 'p'
		// deviceNumMax is the maximum value for trailing numbers on block device name.
		deviceNumMax = 6
		// devicePrefix is the prefix for device names specified when creating volumes.
		devicePrefix = "/dev/sd"
		// renamedDevicePrefix is the prefix for device names after they have
		// been renamed. This should replace "devicePrefix" in the device name
		// when recording the block device info in state.
		renamedDevicePrefix = "xvd"
	)
	var n int
	letterRepeats := 1
	if numbers {
		letterRepeats = deviceNumMax
	}
	return func() (string, string, error) {
		letter := deviceLetterMin + (n / letterRepeats)
		if letter > deviceLetterMax {
			return "", "", errTooManyVolumes
		}
		deviceName := devicePrefix + string(letter)
		if numbers {
			deviceName += string('1' + (n % deviceNumMax))
		}
		n++
		realDeviceName := renamedDevicePrefix + deviceName[len(devicePrefix):]
		return deviceName, realDeviceName, nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package local

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"runtime"

	"launchpad.net/juju-core/container/kvm"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/utils"
	"launchpad.net/juju-core/version"
)

var notLinuxError = errors.New("The local provider is currently only available for Linux")

const installMongodUbuntu = "MongoDB server must be installed to enable the local provider:"
const aptAddRepositoryJujuStable = `
    sudo apt-add-repository ppa:juju/stable   # required for MongoDB SSL support
    sudo apt-get update`
const aptGetInstallMongodbServer = `
    sudo apt-get install mongodb-server`

const installMongodGeneric = `
MongoDB server must be installed to enable the local provider.
Please consult your operating system distribution's documentation
for instructions on installing the MongoDB server. Juju requires
a MongoDB server built with SSL support.
`

const installLxcUbuntu = `
Linux Containers (LXC) userspace tools must be
installed to enable the local provider:

    sudo apt-get install lxc`

const installLxcGeneric = `
Linux Containers (LXC) userspace tools must be installed to enable the
local provider. Please consult your operating system distribution's
documentation for instructions on installing the LXC userspace tools.`

const errUnsupportedOS = `Unsupported operating system: %s
The local provider is currently only available for Linux`

const kvmNeedsUbuntu = `Sorry, but KVM support with the local provider is only supported
on the Ubuntu OS.`

const kvmNotSupported = `KVM is not currently supported with the current settings.
You could try running 'kvm-ok' yourself as root to get the full rationale as to
why it isn't supported, or potentially some BIOS settings to change to enable
KVM support.`

const neetToInstallKVMOk = `kvm-ok is not installed. Please install the cpu-checker package.
    sudo apt-get install cpu-checker`

// mongodPath is the path to "mongod", the MongoDB server.
// This is a variable only to support unit testing.
var mongodPath = "/usr/bin/mongod"

// lxclsPath is the path to "lxc-ls", an LXC userspace tool
// we check the presence of to determine whether the
// tools are installed. This is a variable only to support
// unit testing.
var lxclsPath = "lxc-ls"

// The operating system the process is running in.
// This is a variable only to support unit testing.
var goos = runtime.GOOS

// VerifyPrerequisites verifies the prerequisites of
// the local machine (machine 0) for running the local
// provider.
func VerifyPrerequisites(containerType instance.ContainerType) error {
	if goos != "linux" {
		return fmt.Errorf(errUnsupportedOS, goos)
	}
	if err := verifyMongod(); err != nil {
		return err
	}
	switch containerType {
	case instance.LXC:
		return verifyLxc()
	case instance.KVM:
		return verifyKvm()
	}
	return fmt.Errorf("Unknown container type specified in the config.")
}

func verifyMongod() error {
	if _, err := os.Stat(mongodPath); err != nil {
		if os.IsNotExist(err) {
			return wrapMongodNotExist(err)
		} else {
			return err
		}
	}
	// TODO(axw) verify version/SSL capabilities
	return nil
}

func verifyLxc() error {
	_, err := exec.LookPath(lxclsPath)
	if err != nil {
		return wrapLxcNotFound(err)
	}
	return nil
}

func wrapMongodNotExist(err error) error {
	if utils.IsUbuntu() {
		series := version.Current.Series
		args := []interface{}{err, installMongodUbuntu}
		format := "%v\n%s\n%s"
		if series == "precise" || series == "quantal" {
			format += "%s"
			args = append(args, aptAddRepositoryJujuStable)
		}
		args = append(args, aptGetInstallMongodbServer)
		return fmt.Errorf(format, args...)
	}
	return fmt.Errorf("%v\n%s", err, installMongodGeneric)
}

func wrapLxcNotFound(err error) error {
	if utils.IsUbuntu() {
		return fmt.Errorf("%v\n%s", err, installLxcUbuntu)
	}
	return fmt.Errorf("%v\n%s", err, installLxcGeneric)
}

func verifyKvm() error {
	if !utils.IsUbuntu() {
		return fmt.Errorf(kvmNeedsUbuntu)
	}
	supported, err := kvm.IsKVMSupported()
	if err != nil {
		// Missing the kvm-ok package.
		return fmt.Errorf(neetToInstallKVMOk)
	}
	if !supported {
		return fmt.Errorf(kvmNotSupported)
	}
	// Check for other packages needed.
	// TODO: also check for:
	//   virsh from libvirt-bin
	//   uvt-kvm from uvtool-libvirt
	//   something from the kvm package
	return nil
}
<commit_msg>Add checking for required packages.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package local

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"runtime"

	"launchpad.net/juju-core/container/kvm"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/utils"
	"launchpad.net/juju-core/version"
)

var notLinuxError = errors.New("The local provider is currently only available for Linux")

const installMongodUbuntu = "MongoDB server must be installed to enable the local provider:"
const aptAddRepositoryJujuStable = `
    sudo apt-add-repository ppa:juju/stable   # required for MongoDB SSL support
    sudo apt-get update`
const aptGetInstallMongodbServer = `
    sudo apt-get install mongodb-server`

const installMongodGeneric = `
MongoDB server must be installed to enable the local provider.
Please consult your operating system distribution's documentation
for instructions on installing the MongoDB server. Juju requires
a MongoDB server built with SSL support.
`

const installLxcUbuntu = `
Linux Containers (LXC) userspace tools must be
installed to enable the local provider:

    sudo apt-get install lxc`

const installLxcGeneric = `
Linux Containers (LXC) userspace tools must be installed to enable the
local provider. Please consult your operating system distribution's
documentation for instructions on installing the LXC userspace tools.`

const errUnsupportedOS = `Unsupported operating system: %s
The local provider is currently only available for Linux`

const kvmNeedsUbuntu = `Sorry, but KVM support with the local provider is only supported
on the Ubuntu OS.`

const kvmNotSupported = `KVM is not currently supported with the current settings.
You could try running 'kvm-ok' yourself as root to get the full rationale as to
why it isn't supported, or potentially some BIOS settings to change to enable
KVM support.`

const neetToInstallKVMOk = `kvm-ok is not installed. Please install the cpu-checker package.
    sudo apt-get install cpu-checker`

const missingKVMDeps = `Some required packages are missing for KVM to work:

    sudo apt-get install %s
`

// mongodPath is the path to "mongod", the MongoDB server.
// This is a variable only to support unit testing.
var mongodPath = "/usr/bin/mongod"

// lxclsPath is the path to "lxc-ls", an LXC userspace tool
// we check the presence of to determine whether the
// tools are installed. This is a variable only to support
// unit testing.
var lxclsPath = "lxc-ls"

// The operating system the process is running in.
// This is a variable only to support unit testing.
var goos = runtime.GOOS

// VerifyPrerequisites verifies the prerequisites of
// the local machine (machine 0) for running the local
// provider.
func VerifyPrerequisites(containerType instance.ContainerType) error {
	if goos != "linux" {
		return fmt.Errorf(errUnsupportedOS, goos)
	}
	if err := verifyMongod(); err != nil {
		return err
	}
	switch containerType {
	case instance.LXC:
		return verifyLxc()
	case instance.KVM:
		return verifyKvm()
	}
	return fmt.Errorf("Unknown container type specified in the config.")
}

func verifyMongod() error {
	if _, err := os.Stat(mongodPath); err != nil {
		if os.IsNotExist(err) {
			return wrapMongodNotExist(err)
		} else {
			return err
		}
	}
	// TODO(axw) verify version/SSL capabilities
	return nil
}

func verifyLxc() error {
	_, err := exec.LookPath(lxclsPath)
	if err != nil {
		return wrapLxcNotFound(err)
	}
	return nil
}

func wrapMongodNotExist(err error) error {
	if utils.IsUbuntu() {
		series := version.Current.Series
		args := []interface{}{err, installMongodUbuntu}
		format := "%v\n%s\n%s"
		if series == "precise" || series == "quantal" {
			format += "%s"
			args = append(args, aptAddRepositoryJujuStable)
		}
		args = append(args, aptGetInstallMongodbServer)
		return fmt.Errorf(format, args...)
	}
	return fmt.Errorf("%v\n%s", err, installMongodGeneric)
}

func wrapLxcNotFound(err error) error {
	if utils.IsUbuntu() {
		return fmt.Errorf("%v\n%s", err, installLxcUbuntu)
	}
	return fmt.Errorf("%v\n%s", err, installLxcGeneric)
}

func verifyKvm() error {
	if !utils.IsUbuntu() {
		return fmt.Errorf(kvmNeedsUbuntu)
	}
	supported, err := kvm.IsKVMSupported()
	if err != nil {
		// Missing the kvm-ok package.
		return fmt.Errorf(neetToInstallKVMOk)
	}
	if !supported {
		return fmt.Errorf(kvmNotSupported)
	}
	// Check for other packages needed.
	packagesNeeded = []string{"libvirt-bin", "uvtool-libvirt", "kvm"}
	toInstall = []string{}
	for _, pkg := range packagesNeeded {
		if !utils.IsPackageInstalled(pkg) {
			toInstall = append(toInstall, pkg)
		}
	}
	if len(toInstall) > 0 {
		return fmt.Errorf(missingKVMDeps, strings.Join(toInstall, " "))
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// ClientConnectionConfiguration contains details for constructing a client.
type ClientConnectionConfiguration struct {
	// kubeconfig is the path to a KubeConfig file.
	Kubeconfig string
	// acceptContentTypes defines the Accept header sent by clients when connecting to a server, overriding the
	// default value of 'application/json'. This field will control all connections to the server used by a particular
	// client.
	AcceptContentTypes string
	// contentType is the content type used when sending data to the server from this client.
	ContentType string
	// qps controls the number of queries per second allowed for this connection.
	QPS float32
	// burst allows extra queries to accumulate when a client is exceeding its rate.
	Burst int32
}

// LeaderElectionConfiguration defines the configuration of leader election
// clients for components that can run with leader election enabled.
type LeaderElectionConfiguration struct {
	// leaderElect enables a leader election client to gain leadership
	// before executing the main loop. Enable this when running replicated
	// components for high availability.
	LeaderElect bool
	// leaseDuration is the duration that non-leader candidates will wait
	// after observing a leadership renewal until attempting to acquire
	// leadership of a led but unrenewed leader slot. This is effectively the
	// maximum duration that a leader can be stopped before it is replaced
	// by another candidate. This is only applicable if leader election is
	// enabled.
	LeaseDuration metav1.Duration
	// renewDeadline is the interval between attempts by the acting master to
	// renew a leadership slot before it stops leading. This must be less
	// than or equal to the lease duration. This is only applicable if leader
	// election is enabled.
	RenewDeadline metav1.Duration
	// retryPeriod is the duration the clients should wait between attempting
	// acquisition and renewal of a leadership. This is only applicable if
	// leader election is enabled.
	RetryPeriod metav1.Duration
	// resourceLock indicates the resource object type that will be used to lock
	// during leader election cycles.
	ResourceLock string
	// resourceName indicates the name of resource object that will be used to lock
	// during leader election cycles.
	ResourceName string
	// resourceName indicates the namespace of resource object that will be used to lock
	// during leader election cycles.
	ResourceNamespace string
}

// DebuggingConfiguration holds configuration for Debugging related features.
type DebuggingConfiguration struct {
	// enableProfiling enables profiling via web interface host:port/debug/pprof/
	EnableProfiling bool
	// enableContentionProfiling enables lock contention profiling, if
	// enableProfiling is true.
	EnableContentionProfiling bool
}

// LoggingConfiguration contains logging options
// Refer [Logs Options](https://github.com/kubernetes/component-base/blob/master/logs/options.go) for more information.
type LoggingConfiguration struct {
	// Format Flag specifies the structure of log messages.
	// default value of format is `text`
	Format string
	// [Experimental] When enabled prevents logging of fields tagged as sensitive (passwords, keys, tokens).
	// Runtime log sanitization may introduce significant computation overhead and therefore should not be enabled in production.`)
	Sanitization bool
}
<commit_msg>Modify LeaderElectionConfiguration .ResourceNamespace comment<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// ClientConnectionConfiguration contains details for constructing a client.
type ClientConnectionConfiguration struct {
	// kubeconfig is the path to a KubeConfig file.
	Kubeconfig string
	// acceptContentTypes defines the Accept header sent by clients when connecting to a server, overriding the
	// default value of 'application/json'. This field will control all connections to the server used by a particular
	// client.
	AcceptContentTypes string
	// contentType is the content type used when sending data to the server from this client.
	ContentType string
	// qps controls the number of queries per second allowed for this connection.
	QPS float32
	// burst allows extra queries to accumulate when a client is exceeding its rate.
	Burst int32
}

// LeaderElectionConfiguration defines the configuration of leader election
// clients for components that can run with leader election enabled.
type LeaderElectionConfiguration struct {
	// leaderElect enables a leader election client to gain leadership
	// before executing the main loop. Enable this when running replicated
	// components for high availability.
	LeaderElect bool
	// leaseDuration is the duration that non-leader candidates will wait
	// after observing a leadership renewal until attempting to acquire
	// leadership of a led but unrenewed leader slot. This is effectively the
	// maximum duration that a leader can be stopped before it is replaced
	// by another candidate. This is only applicable if leader election is
	// enabled.
	LeaseDuration metav1.Duration
	// renewDeadline is the interval between attempts by the acting master to
	// renew a leadership slot before it stops leading. This must be less
	// than or equal to the lease duration. This is only applicable if leader
	// election is enabled.
	RenewDeadline metav1.Duration
	// retryPeriod is the duration the clients should wait between attempting
	// acquisition and renewal of a leadership. This is only applicable if
	// leader election is enabled.
	RetryPeriod metav1.Duration
	// resourceLock indicates the resource object type that will be used to lock
	// during leader election cycles.
	ResourceLock string
	// resourceName indicates the name of resource object that will be used to lock
	// during leader election cycles.
	ResourceName string
	// resourceNamespace indicates the namespace of resource object that will be used to lock
	// during leader election cycles.
	ResourceNamespace string
}

// DebuggingConfiguration holds configuration for Debugging related features.
type DebuggingConfiguration struct {
	// enableProfiling enables profiling via web interface host:port/debug/pprof/
	EnableProfiling bool
	// enableContentionProfiling enables lock contention profiling, if
	// enableProfiling is true.
	EnableContentionProfiling bool
}

// LoggingConfiguration contains logging options
// Refer [Logs Options](https://github.com/kubernetes/component-base/blob/master/logs/options.go) for more information.
type LoggingConfiguration struct {
	// Format Flag specifies the structure of log messages.
	// default value of format is `text`
	Format string
	// [Experimental] When enabled prevents logging of fields tagged as sensitive (passwords, keys, tokens).
	// Runtime log sanitization may introduce significant computation overhead and therefore should not be enabled in production.`)
	Sanitization bool
}
<|endoftext|>


<commit_before>package matchers

var (
	TypeEpub   = newType("epub", "application/epub+zip")
	TypeZip    = newType("zip", "application/zip")
	TypeTar    = newType("tar", "application/x-tar")
	TypeRar    = newType("rar", "application/x-rar-compressed")
	TypeGz     = newType("gz", "application/gzip")
	TypeBz2    = newType("bz2", "application/x-bzip2")
	Type7z     = newType("7z", "application/x-7z-compressed")
	TypeXz     = newType("xz", "application/x-xz")
	TypePdf    = newType("pdf", "application/pdf")
	TypeExe    = newType("exe", "application/x-msdownload")
	TypeSwf    = newType("swf", "application/x-shockwave-flash")
	TypeRtf    = newType("rtf", "application/rtf")
	TypeEot    = newType("eot", "application/octet-stream")
	TypePs     = newType("ps", "application/postscript")
	TypeSqlite = newType("sqlite", "application/x-sqlite3")
	TypeNes    = newType("nes", "application/x-nintendo-nes-rom")
	TypeCrx    = newType("crx", "application/x-google-chrome-extension")
	TypeCab    = newType("cab", "application/vnd.ms-cab-compressed")
	TypeDeb    = newType("deb", "application/x-deb")
	TypeAr     = newType("ar", "application/x-unix-archive")
	TypeZ      = newType("Z", "application/x-compress")
	TypeLz     = newType("lz", "application/x-lzip")
	TypeRpm    = newType("rpm", "application/x-rpm")
	TypeElf    = newType("elf", "application/x-executable")
	TypeDcm    = newType("dcm", "application/dicom")
)

var Archive = Map{
	TypeEpub:   Epub,
	TypeZip:    Zip,
	TypeTar:    Tar,
	TypeRar:    Rar,
	TypeGz:     Gz,
	TypeBz2:    Bz2,
	Type7z:     SevenZ,
	TypeXz:     Xz,
	TypePdf:    Pdf,
	TypeExe:    Exe,
	TypeSwf:    Swf,
	TypeRtf:    Rtf,
	TypeEot:    Eot,
	TypePs:     Ps,
	TypeSqlite: Sqlite,
	TypeNes:    Nes,
	TypeCrx:    Crx,
	TypeCab:    Cab,
	TypeDeb:    Deb,
	TypeAr:     Ar,
	TypeZ:      Z,
	TypeLz:     Lz,
	TypeRpm:    Rpm,
	TypeElf:    Elf,
	TypeDcm:    Dcm,
}

func Epub(buf []byte) bool {
	return len(buf) > 57 &&
		buf[0] == 0x50 && buf[1] == 0x4B && buf[2] == 0x3 && buf[3] == 0x4 &&
		buf[30] == 0x6D && buf[31] == 0x69 && buf[32] == 0x6D && buf[33] == 0x65 &&
		buf[34] == 0x74 && buf[35] == 0x79 && buf[36] == 0x70 && buf[37] == 0x65 &&
		buf[38] == 0x61 && buf[39] == 0x70 && buf[40] == 0x70 && buf[41] == 0x6C &&
		buf[42] == 0x69 && buf[43] == 0x63 && buf[44] == 0x61 && buf[45] == 0x74 &&
		buf[46] == 0x69 && buf[47] == 0x6F && buf[48] == 0x6E && buf[49] == 0x2F &&
		buf[50] == 0x65 && buf[51] == 0x70 && buf[52] == 0x75 && buf[53] == 0x62 &&
		buf[54] == 0x2B && buf[55] == 0x7A && buf[56] == 0x69 && buf[57] == 0x70
}

func Zip(buf []byte) bool {
	return len(buf) > 3 &&
		buf[0] == 0x50 && buf[1] == 0x4B &&
		(buf[2] == 0x3 || buf[2] == 0x5 || buf[2] == 0x7) &&
		(buf[3] == 0x4 || buf[3] == 0x6 || buf[3] == 0x8)
}

func Tar(buf []byte) bool {
	return len(buf) > 261 &&
		buf[257] == 0x75 && buf[258] == 0x73 &&
		buf[259] == 0x74 && buf[260] == 0x61 &&
		buf[261] == 0x72
}

func Rar(buf []byte) bool {
	return len(buf) > 6 &&
		buf[0] == 0x52 && buf[1] == 0x61 && buf[2] == 0x72 &&
		buf[3] == 0x21 && buf[4] == 0x1A && buf[5] == 0x7 &&
		(buf[6] == 0x0 || buf[6] == 0x1)
}

func Gz(buf []byte) bool {
	return len(buf) > 2 &&
		buf[0] == 0x1F && buf[1] == 0x8B && buf[2] == 0x8
}

func Bz2(buf []byte) bool {
	return len(buf) > 2 &&
		buf[0] == 0x42 && buf[1] == 0x5A && buf[2] == 0x68
}

func SevenZ(buf []byte) bool {
	return len(buf) > 5 &&
		buf[0] == 0x37 && buf[1] == 0x7A && buf[2] == 0xBC &&
		buf[3] == 0xAF && buf[4] == 0x27 && buf[5] == 0x1C
}

func Pdf(buf []byte) bool {
	return len(buf) > 3 &&
		buf[0] == 0x25 && buf[1] == 0x50 &&
		buf[2] == 0x44 && buf[3] == 0x46
}

func Exe(buf []byte) bool {
	return len(buf) > 1 &&
		buf[0] == 0x4D && buf[1] == 0x5A
}

func Swf(buf []byte) bool {
	return len(buf) > 2 &&
		(buf[0] == 0x43 || buf[0] == 0x46) &&
		buf[1] == 0x57 && buf[2] == 0x53
}

func Rtf(buf []byte) bool {
	return len(buf) > 4 &&
		buf[0] == 0x7B && buf[1] == 0x5C &&
		buf[2] == 0x72 && buf[3] == 0x74 &&
		buf[4] == 0x66
}

func Nes(buf []byte) bool {
	return len(buf) > 3 &&
		buf[0] == 0x4E && buf[1] == 0x45 &&
		buf[2] == 0x53 && buf[3] == 0x1A
}

func Crx(buf []byte) bool {
	return len(buf) > 3 &&
		buf[0] == 0x43 && buf[1] == 0x72 &&
		buf[2] == 0x32 && buf[3] == 0x34
}

func Cab(buf []byte) bool {
	return len(buf) > 3 &&
		((buf[0] == 0x4D && buf[1] == 0x53 && buf[2] == 0x43 && buf[3] == 0x46) ||
			(buf[0] == 0x49 && buf[1] == 0x53 && buf[2] == 0x63 && buf[3] == 0x28))
}

func Eot(buf []byte) bool {
	return len(buf) > 35 &&
		buf[34] == 0x4C && buf[35] == 0x50 &&
		((buf[8] == 0x02 && buf[9] == 0x00 &&
			buf[10] == 0x01) || (buf[8] == 0x01 &&
			buf[9] == 0x00 && buf[10] == 0x00) ||
			(buf[8] == 0x02 && buf[9] == 0x00 &&
				buf[10] == 0x02))
}

func Ps(buf []byte) bool {
	return len(buf) > 1 &&
		buf[0] == 0x25 && buf[1] == 0x21
}

func Xz(buf []byte) bool {
	return len(buf) > 5 &&
		buf[0] == 0xFD && buf[1] == 0x37 &&
		buf[2] == 0x7A && buf[3] == 0x58 &&
		buf[4] == 0x5A && buf[5] == 0x00
}

func Sqlite(buf []byte) bool {
	return len(buf) > 3 &&
		buf[0] == 0x53 && buf[1] == 0x51 &&
		buf[2] == 0x4C && buf[3] == 0x69
}

func Deb(buf []byte) bool {
	return len(buf) > 20 &&
		buf[0] == 0x21 && buf[1] == 0x3C && buf[2] == 0x61 &&
		buf[3] == 0x72 && buf[4] == 0x63 && buf[5] == 0x68 &&
		buf[6] == 0x3E && buf[7] == 0x0A && buf[8] == 0x64 &&
		buf[9] == 0x65 && buf[10] == 0x62 && buf[11] == 0x69 &&
		buf[12] == 0x61 && buf[13] == 0x6E && buf[14] == 0x2D &&
		buf[15] == 0x62 && buf[16] == 0x69 && buf[17] == 0x6E &&
		buf[18] == 0x61 && buf[19] == 0x72 && buf[20] == 0x79
}

func Ar(buf []byte) bool {
	return len(buf) > 6 &&
		buf[0] == 0x21 && buf[1] == 0x3C &&
		buf[2] == 0x61 && buf[3] == 0x72 &&
		buf[4] == 0x63 && buf[5] == 0x68 &&
		buf[6] == 0x3E
}

func Z(buf []byte) bool {
	return len(buf) > 1 &&
		((buf[0] == 0x1F && buf[1] == 0xA0) ||
			(buf[0] == 0x1F && buf[1] == 0x9D))
}

func Lz(buf []byte) bool {
	return len(buf) > 3 &&
		buf[0] == 0x4C && buf[1] == 0x5A &&
		buf[2] == 0x49 && buf[3] == 0x50
}

func Rpm(buf []byte) bool {
	return len(buf) > 96 &&
		buf[0] == 0xED && buf[1] == 0xAB &&
		buf[2] == 0xEE && buf[3] == 0xDB
}

func Elf(buf []byte) bool {
	return len(buf) > 52 &&
		buf[0] == 0x7F && buf[1] == 0x45 &&
		buf[2] == 0x4C && buf[3] == 0x46
}

func Dcm(buf []byte) bool {
	return len(buf) > 131 &&
		buf[128] == 0x44 && buf[129] == 0x49 &&
		buf[130] == 0x43 && buf[131] == 0x4D
}
<commit_msg>Added support for .iso files<commit_after>package matchers

var (
	TypeEpub   = newType("epub", "application/epub+zip")
	TypeZip    = newType("zip", "application/zip")
	TypeTar    = newType("tar", "application/x-tar")
	TypeRar    = newType("rar", "application/x-rar-compressed")
	TypeGz     = newType("gz", "application/gzip")
	TypeBz2    = newType("bz2", "application/x-bzip2")
	Type7z     = newType("7z", "application/x-7z-compressed")
	TypeXz     = newType("xz", "application/x-xz")
	TypePdf    = newType("pdf", "application/pdf")
	TypeExe    = newType("exe", "application/x-msdownload")
	TypeSwf    = newType("swf", "application/x-shockwave-flash")
	TypeRtf    = newType("rtf", "application/rtf")
	TypeEot    = newType("eot", "application/octet-stream")
	TypePs     = newType("ps", "application/postscript")
	TypeSqlite = newType("sqlite", "application/x-sqlite3")
	TypeNes    = newType("nes", "application/x-nintendo-nes-rom")
	TypeCrx    = newType("crx", "application/x-google-chrome-extension")
	TypeCab    = newType("cab", "application/vnd.ms-cab-compressed")
	TypeDeb    = newType("deb", "application/x-deb")
	TypeAr     = newType("ar", "application/x-unix-archive")
	TypeZ      = newType("Z", "application/x-compress")
	TypeLz     = newType("lz", "application/x-lzip")
	TypeRpm    = newType("rpm", "application/x-rpm")
	TypeElf    = newType("elf", "application/x-executable")
	TypeDcm    = newType("dcm", "application/dicom")
	TypeIso    = newType("iso", "application/iso")
)

var Archive = Map{
	TypeEpub:   Epub,
	TypeZip:    Zip,
	TypeTar:    Tar,
	TypeRar:    Rar,
	TypeGz:     Gz,
	TypeBz2:    Bz2,
	Type7z:     SevenZ,
	TypeXz:     Xz,
	TypePdf:    Pdf,
	TypeExe:    Exe,
	TypeSwf:    Swf,
	TypeRtf:    Rtf,
	TypeEot:    Eot,
	TypePs:     Ps,
	TypeSqlite: Sqlite,
	TypeNes:    Nes,
	TypeCrx:    Crx,
	TypeCab:    Cab,
	TypeDeb:    Deb,
	TypeAr:     Ar,
	TypeZ:      Z,
	TypeLz:     Lz,
	TypeRpm:    Rpm,
	TypeElf:    Elf,
	TypeDcm:    Dcm,
	TypeIso:    Iso,
}

func Epub(buf []byte) bool {
	return len(buf) > 57 &&
		buf[0] == 0x50 && buf[1] == 0x4B && buf[2] == 0x3 && buf[3] == 0x4 &&
		buf[30] == 0x6D && buf[31] == 0x69 && buf[32] == 0x6D && buf[33] == 0x65 &&
		buf[34] == 0x74 && buf[35] == 0x79 && buf[36] == 0x70 && buf[37] == 0x65 &&
		buf[38] == 0x61 && buf[39] == 0x70 && buf[40] == 0x70 && buf[41] == 0x6C &&
		buf[42] == 0x69 && buf[43] == 0x63 && buf[44] == 0x61 && buf[45] == 0x74 &&
		buf[46] == 0x69 && buf[47] == 0x6F && buf[48] == 0x6E && buf[49] == 0x2F &&
		buf[50] == 0x65 && buf[51] == 0x70 && buf[52] == 0x75 && buf[53] == 0x62 &&
		buf[54] == 0x2B && buf[55] == 0x7A && buf[56] == 0x69 && buf[57] == 0x70
}

func Zip(buf []byte) bool {
	return len(buf) > 3 &&
		buf[0] == 0x50 && buf[1] == 0x4B &&
		(buf[2] == 0x3 || buf[2] == 0x5 || buf[2] == 0x7) &&
		(buf[3] == 0x4 || buf[3] == 0x6 || buf[3] == 0x8)
}

func Tar(buf []byte) bool {
	return len(buf) > 261 &&
		buf[257] == 0x75 && buf[258] == 0x73 &&
		buf[259] == 0x74 && buf[260] == 0x61 &&
		buf[261] == 0x72
}

func Rar(buf []byte) bool {
	return len(buf) > 6 &&
		buf[0] == 0x52 && buf[1] == 0x61 && buf[2] == 0x72 &&
		buf[3] == 0x21 && buf[4] == 0x1A && buf[5] == 0x7 &&
		(buf[6] == 0x0 || buf[6] == 0x1)
}

func Gz(buf []byte) bool {
	return len(buf) > 2 &&
		buf[0] == 0x1F && buf[1] == 0x8B && buf[2] == 0x8
}

func Bz2(buf []byte) bool {
	return len(buf) > 2 &&
		buf[0] == 0x42 && buf[1] == 0x5A && buf[2] == 0x68
}

func SevenZ(buf []byte) bool {
	return len(buf) > 5 &&
		buf[0] == 0x37 && buf[1] == 0x7A && buf[2] == 0xBC &&
		buf[3] == 0xAF && buf[4] == 0x27 && buf[5] == 0x1C
}

func Pdf(buf []byte) bool {
	return len(buf) > 3 &&
		buf[0] == 0x25 && buf[1] == 0x50 &&
		buf[2] == 0x44 && buf[3] == 0x46
}

func Exe(buf []byte) bool {
	return len(buf) > 1 &&
		buf[0] == 0x4D && buf[1] == 0x5A
}

func Swf(buf []byte) bool {
	return len(buf) > 2 &&
		(buf[0] == 0x43 || buf[0] == 0x46) &&
		buf[1] == 0x57 && buf[2] == 0x53
}

func Rtf(buf []byte) bool {
	return len(buf) > 4 &&
		buf[0] == 0x7B && buf[1] == 0x5C &&
		buf[2] == 0x72 && buf[3] == 0x74 &&
		buf[4] == 0x66
}

func Nes(buf []byte) bool {
	return len(buf) > 3 &&
		buf[0] == 0x4E && buf[1] == 0x45 &&
		buf[2] == 0x53 && buf[3] == 0x1A
}

func Crx(buf []byte) bool {
	return len(buf) > 3 &&
		buf[0] == 0x43 && buf[1] == 0x72 &&
		buf[2] == 0x32 && buf[3] == 0x34
}

func Cab(buf []byte) bool {
	return len(buf) > 3 &&
		((buf[0] == 0x4D && buf[1] == 0x53 && buf[2] == 0x43 && buf[3] == 0x46) ||
			(buf[0] == 0x49 && buf[1] == 0x53 && buf[2] == 0x63 && buf[3] == 0x28))
}

func Eot(buf []byte) bool {
	return len(buf) > 35 &&
		buf[34] == 0x4C && buf[35] == 0x50 &&
		((buf[8] == 0x02 && buf[9] == 0x00 &&
			buf[10] == 0x01) || (buf[8] == 0x01 &&
			buf[9] == 0x00 && buf[10] == 0x00) ||
			(buf[8] == 0x02 && buf[9] == 0x00 &&
				buf[10] == 0x02))
}

func Ps(buf []byte) bool {
	return len(buf) > 1 &&
		buf[0] == 0x25 && buf[1] == 0x21
}

func Xz(buf []byte) bool {
	return len(buf) > 5 &&
		buf[0] == 0xFD && buf[1] == 0x37 &&
		buf[2] == 0x7A && buf[3] == 0x58 &&
		buf[4] == 0x5A && buf[5] == 0x00
}

func Sqlite(buf []byte) bool {
	return len(buf) > 3 &&
		buf[0] == 0x53 && buf[1] == 0x51 &&
		buf[2] == 0x4C && buf[3] == 0x69
}

func Deb(buf []byte) bool {
	return len(buf) > 20 &&
		buf[0] == 0x21 && buf[1] == 0x3C && buf[2] == 0x61 &&
		buf[3] == 0x72 && buf[4] == 0x63 && buf[5] == 0x68 &&
		buf[6] == 0x3E && buf[7] == 0x0A && buf[8] == 0x64 &&
		buf[9] == 0x65 && buf[10] == 0x62 && buf[11] == 0x69 &&
		buf[12] == 0x61 && buf[13] == 0x6E && buf[14] == 0x2D &&
		buf[15] == 0x62 && buf[16] == 0x69 && buf[17] == 0x6E &&
		buf[18] == 0x61 && buf[19] == 0x72 && buf[20] == 0x79
}

func Ar(buf []byte) bool {
	return len(buf) > 6 &&
		buf[0] == 0x21 && buf[1] == 0x3C &&
		buf[2] == 0x61 && buf[3] == 0x72 &&
		buf[4] == 0x63 && buf[5] == 0x68 &&
		buf[6] == 0x3E
}

func Z(buf []byte) bool {
	return len(buf) > 1 &&
		((buf[0] == 0x1F && buf[1] == 0xA0) ||
			(buf[0] == 0x1F && buf[1] == 0x9D))
}

func Lz(buf []byte) bool {
	return len(buf) > 3 &&
		buf[0] == 0x4C && buf[1] == 0x5A &&
		buf[2] == 0x49 && buf[3] == 0x50
}

func Rpm(buf []byte) bool {
	return len(buf) > 96 &&
		buf[0] == 0xED && buf[1] == 0xAB &&
		buf[2] == 0xEE && buf[3] == 0xDB
}

func Elf(buf []byte) bool {
	return len(buf) > 52 &&
		buf[0] == 0x7F && buf[1] == 0x45 &&
		buf[2] == 0x4C && buf[3] == 0x46
}

func Dcm(buf []byte) bool {
	return len(buf) > 131 &&
		buf[128] == 0x44 && buf[129] == 0x49 &&
		buf[130] == 0x43 && buf[131] == 0x4D
}

func Iso(buf []byte) bool {
	return len(buf) > 32773 &&
		buf[32769] == 0x43 && buf[32770] == 0x44 &&
		buf[32771] == 0x30 && buf[32772] == 0x30 &&
		buf[32773] == 0x31
}
<|endoftext|>


<commit_before>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"fmt"
	"strings"

	"github.com/fsouza/go-dockerclient"
	"github.com/tsuru/config"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/docker-cluster/storage"
	"github.com/tsuru/tsuru/app"
	"github.com/tsuru/tsuru/db"
	dbStorage "github.com/tsuru/tsuru/db/storage"
	"github.com/tsuru/tsuru/log"
	"github.com/tsuru/tsuru/provision"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type appImages struct {
	AppName string `bson:"_id"`
	Images  []string
	Count   int
}

func migrateImages() error {
	registry, _ := config.GetString("docker:registry")
	if registry != "" {
		registry += "/"
	}
	repoNamespace, err := config.GetString("docker:repository-namespace")
	if err != nil {
		return err
	}
	apps, err := app.List(nil)
	if err != nil {
		return err
	}
	dcluster := dockerCluster()
	for _, app := range apps {
		oldImage := registry + repoNamespace + "/" + app.Name
		newImage := registry + repoNamespace + "/app-" + app.Name
		opts := docker.TagImageOptions{Repo: newImage, Force: true}
		err = dcluster.TagImage(oldImage, opts)
		var baseErr error
		if nodeErr, ok := err.(cluster.DockerNodeError); ok {
			baseErr = nodeErr.BaseError()
		}
		if err != nil && err != storage.ErrNoSuchImage && baseErr != docker.ErrNoSuchImage {
			return err
		}
		if registry != "" {
			pushOpts := docker.PushImageOptions{Name: newImage}
			err = dcluster.PushImage(pushOpts, docker.AuthConfiguration{})
			if err != nil {
				return err
			}
		}
		err = updateContainers(bson.M{"appname": app.Name}, bson.M{"$set": bson.M{"image": newImage}})
		if err != nil {
			return err
		}
	}
	return nil
}

// getBuildImage returns the image name from app or plaftorm.
// the platform image will be returned if:
// * there are no containers;
// * the container have an empty image name;
// * the deploy number is multiple of 10.
// in all other cases the app image name will be returne.
func getBuildImage(app provision.App) string {
	if usePlatformImage(app) {
		return platformImageName(app.GetPlatform())
	}
	appImageName, err := appCurrentImageName(app.GetName())
	if err != nil {
		return platformImageName(app.GetPlatform())
	}
	return appImageName
}

func appImagesColl() (*dbStorage.Collection, error) {
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	name, err := config.GetString("docker:collection")
	if err != nil {
		return nil, err
	}
	return conn.Collection(fmt.Sprintf("%s_app_image", name)), nil
}

func imageCustomDataColl() (*dbStorage.Collection, error) {
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	name, err := config.GetString("docker:collection")
	if err != nil {
		return nil, err
	}
	return conn.Collection(fmt.Sprintf("%s_image_custom_data", name)), nil
}

func saveImageCustomData(imageName string, customData map[string]interface{}) error {
	coll, err := imageCustomDataColl()
	if err != nil {
		return err
	}
	defer coll.Close()
	return coll.Insert(bson.M{"_id": imageName, "customdata": customData})
}

func getImageTsuruYamlData(imageName string) (provision.TsuruYamlData, error) {
	var customData struct {
		Customdata provision.TsuruYamlData
	}
	coll, err := imageCustomDataColl()
	if err != nil {
		return customData.Customdata, err
	}
	defer coll.Close()
	err = coll.FindId(imageName).One(&customData)
	return customData.Customdata, err
}

// TODO(cezarsa): This method only exist to keep tsuru compatible with older
// platforms. It should be deprecated in the next major after 0.10.0.
func getImageTsuruYamlDataWithFallback(imageName, appName string) (provision.TsuruYamlData, error) {
	yamlData, err := getImageTsuruYamlData(imageName)
	if err != nil {
		a, err := app.GetByName(appName)
		if err != nil {
			// Ignored error if app not found
			return yamlData, nil
		}
		return a.GetTsuruYamlData()
	}
	return yamlData, nil
}

func appNewImageName(appName string) (string, error) {
	coll, err := appImagesColl()
	if err != nil {
		return "", err
	}
	defer coll.Close()
	var imgs appImages
	dbChange := mgo.Change{
		Update:    bson.M{"$inc": bson.M{"count": 1}},
		ReturnNew: true,
		Upsert:    true,
	}
	_, err = coll.FindId(appName).Apply(dbChange, &imgs)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("%s/app-%s:v%d", basicImageName(), appName, imgs.Count), nil
}

func appCurrentImageName(appName string) (string, error) {
	coll, err := appImagesColl()
	if err != nil {
		return "", err
	}
	defer coll.Close()
	var imgs appImages
	err = coll.FindId(appName).One(&imgs)
	if err != nil {
		log.Errorf("Couldn't find images for app %q, fallback to old image names. Error: %s", appName, err.Error())
		return fmt.Sprintf("%s/app-%s", basicImageName(), appName), nil
	}
	if len(imgs.Images) == 0 {
		return "", fmt.Errorf("no images available for app %q", appName)
	}
	return imgs.Images[len(imgs.Images)-1], nil
}

func appendAppImageName(appName, imageId string) error {
	coll, err := appImagesColl()
	if err != nil {
		return err
	}
	defer coll.Close()
	_, err = coll.UpsertId(appName, bson.M{"$pull": bson.M{"images": imageId}})
	if err != nil {
		return err
	}
	_, err = coll.UpsertId(appName, bson.M{"$push": bson.M{"images": imageId}})
	return err
}

func listAppImages(appName string) ([]string, error) {
	coll, err := appImagesColl()
	if err != nil {
		return nil, err
	}
	defer coll.Close()
	var imgs appImages
	err = coll.FindId(appName).One(&imgs)
	if err != nil {
		return nil, err
	}
	return imgs.Images, nil
}

func listValidAppImages(appName string) ([]string, error) {
	coll, err := appImagesColl()
	if err != nil {
		return nil, err
	}
	defer coll.Close()
	var img appImages
	err = coll.FindId(appName).One(&img)
	if err != nil {
		return nil, err
	}
	historySize := imageHistorySize()
	if len(img.Images) > historySize {
		img.Images = img.Images[len(img.Images)-historySize:]
	}
	return img.Images, nil
}

func isValidAppImage(appName, imageId string) (bool, error) {
	images, err := listValidAppImages(appName)
	if err != nil && err != mgo.ErrNotFound {
		return false, err
	}
	for _, img := range images {
		if img == imageId {
			return true, nil
		}
	}
	return false, nil
}

func imageHistorySize() int {
	imgHistorySize, _ := config.GetInt("docker:image-history-size")
	if imgHistorySize == 0 {
		imgHistorySize = 10
	}
	return imgHistorySize
}

func deleteAllAppImageNames(appName string) error {
	images, err := listAppImages(appName)
	if err != nil {
		return err
	}
	dataColl, err := imageCustomDataColl()
	if err != nil {
		return err
	}
	defer dataColl.Close()
	_, err = dataColl.RemoveAll(bson.M{"_id": bson.M{"$in": images}})
	if err != nil {
		return err
	}
	coll, err := appImagesColl()
	if err != nil {
		return err
	}
	defer coll.Close()
	return coll.RemoveId(appName)
}

func pullAppImageNames(appName string, images []string) error {
	dataColl, err := imageCustomDataColl()
	if err != nil {
		return err
	}
	defer dataColl.Close()
	_, err = dataColl.RemoveAll(bson.M{"_id": bson.M{"$in": images}})
	if err != nil {
		return err
	}
	coll, err := appImagesColl()
	if err != nil {
		return err
	}
	defer coll.Close()
	return coll.UpdateId(appName, bson.M{"$pullAll": bson.M{"images": images}})
}

func platformImageName(platformName string) string {
	return fmt.Sprintf("%s/%s", basicImageName(), platformName)
}

func basicImageName() string {
	parts := make([]string, 0, 2)
	registry, _ := config.GetString("docker:registry")
	if registry != "" {
		parts = append(parts, registry)
	}
	repoNamespace, _ := config.GetString("docker:repository-namespace")
	parts = append(parts, repoNamespace)
	return strings.Join(parts, "/")
}

func usePlatformImage(app provision.App) bool {
	deploys := app.GetDeploys()
	if (deploys != 0 && deploys%10 == 0) || app.GetUpdatePlatform() {
		return true
	}
	c, err := getOneContainerByAppName(app.GetName())
	if err != nil || c.Image == "" {
		return true
	}
	return false
}
<commit_msg>provision/docker: valid image names return empty if no img is available<commit_after>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"fmt"
	"strings"

	"github.com/fsouza/go-dockerclient"
	"github.com/tsuru/config"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/docker-cluster/storage"
	"github.com/tsuru/tsuru/app"
	"github.com/tsuru/tsuru/db"
	dbStorage "github.com/tsuru/tsuru/db/storage"
	"github.com/tsuru/tsuru/log"
	"github.com/tsuru/tsuru/provision"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type appImages struct {
	AppName string `bson:"_id"`
	Images  []string
	Count   int
}

func migrateImages() error {
	registry, _ := config.GetString("docker:registry")
	if registry != "" {
		registry += "/"
	}
	repoNamespace, err := config.GetString("docker:repository-namespace")
	if err != nil {
		return err
	}
	apps, err := app.List(nil)
	if err != nil {
		return err
	}
	dcluster := dockerCluster()
	for _, app := range apps {
		oldImage := registry + repoNamespace + "/" + app.Name
		newImage := registry + repoNamespace + "/app-" + app.Name
		opts := docker.TagImageOptions{Repo: newImage, Force: true}
		err = dcluster.TagImage(oldImage, opts)
		var baseErr error
		if nodeErr, ok := err.(cluster.DockerNodeError); ok {
			baseErr = nodeErr.BaseError()
		}
		if err != nil && err != storage.ErrNoSuchImage && baseErr != docker.ErrNoSuchImage {
			return err
		}
		if registry != "" {
			pushOpts := docker.PushImageOptions{Name: newImage}
			err = dcluster.PushImage(pushOpts, docker.AuthConfiguration{})
			if err != nil {
				return err
			}
		}
		err = updateContainers(bson.M{"appname": app.Name}, bson.M{"$set": bson.M{"image": newImage}})
		if err != nil {
			return err
		}
	}
	return nil
}

// getBuildImage returns the image name from app or plaftorm.
// the platform image will be returned if:
// * there are no containers;
// * the container have an empty image name;
// * the deploy number is multiple of 10.
// in all other cases the app image name will be returne.
func getBuildImage(app provision.App) string {
	if usePlatformImage(app) {
		return platformImageName(app.GetPlatform())
	}
	appImageName, err := appCurrentImageName(app.GetName())
	if err != nil {
		return platformImageName(app.GetPlatform())
	}
	return appImageName
}

func appImagesColl() (*dbStorage.Collection, error) {
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	name, err := config.GetString("docker:collection")
	if err != nil {
		return nil, err
	}
	return conn.Collection(fmt.Sprintf("%s_app_image", name)), nil
}

func imageCustomDataColl() (*dbStorage.Collection, error) {
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	name, err := config.GetString("docker:collection")
	if err != nil {
		return nil, err
	}
	return conn.Collection(fmt.Sprintf("%s_image_custom_data", name)), nil
}

func saveImageCustomData(imageName string, customData map[string]interface{}) error {
	coll, err := imageCustomDataColl()
	if err != nil {
		return err
	}
	defer coll.Close()
	return coll.Insert(bson.M{"_id": imageName, "customdata": customData})
}

func getImageTsuruYamlData(imageName string) (provision.TsuruYamlData, error) {
	var customData struct {
		Customdata provision.TsuruYamlData
	}
	coll, err := imageCustomDataColl()
	if err != nil {
		return customData.Customdata, err
	}
	defer coll.Close()
	err = coll.FindId(imageName).One(&customData)
	return customData.Customdata, err
}

// TODO(cezarsa): This method only exist to keep tsuru compatible with older
// platforms. It should be deprecated in the next major after 0.10.0.
func getImageTsuruYamlDataWithFallback(imageName, appName string) (provision.TsuruYamlData, error) {
	yamlData, err := getImageTsuruYamlData(imageName)
	if err != nil {
		a, err := app.GetByName(appName)
		if err != nil {
			// Ignored error if app not found
			return yamlData, nil
		}
		return a.GetTsuruYamlData()
	}
	return yamlData, nil
}

func appNewImageName(appName string) (string, error) {
	coll, err := appImagesColl()
	if err != nil {
		return "", err
	}
	defer coll.Close()
	var imgs appImages
	dbChange := mgo.Change{
		Update:    bson.M{"$inc": bson.M{"count": 1}},
		ReturnNew: true,
		Upsert:    true,
	}
	_, err = coll.FindId(appName).Apply(dbChange, &imgs)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("%s/app-%s:v%d", basicImageName(), appName, imgs.Count), nil
}

func appCurrentImageName(appName string) (string, error) {
	coll, err := appImagesColl()
	if err != nil {
		return "", err
	}
	defer coll.Close()
	var imgs appImages
	err = coll.FindId(appName).One(&imgs)
	if err != nil {
		log.Errorf("Couldn't find images for app %q, fallback to old image names. Error: %s", appName, err.Error())
		return fmt.Sprintf("%s/app-%s", basicImageName(), appName), nil
	}
	if len(imgs.Images) == 0 {
		return "", fmt.Errorf("no images available for app %q", appName)
	}
	return imgs.Images[len(imgs.Images)-1], nil
}

func appendAppImageName(appName, imageId string) error {
	coll, err := appImagesColl()
	if err != nil {
		return err
	}
	defer coll.Close()
	_, err = coll.UpsertId(appName, bson.M{"$pull": bson.M{"images": imageId}})
	if err != nil {
		return err
	}
	_, err = coll.UpsertId(appName, bson.M{"$push": bson.M{"images": imageId}})
	return err
}

func listAppImages(appName string) ([]string, error) {
	coll, err := appImagesColl()
	if err != nil {
		return nil, err
	}
	defer coll.Close()
	var imgs appImages
	err = coll.FindId(appName).One(&imgs)
	if err != nil {
		return nil, err
	}
	return imgs.Images, nil
}

func listValidAppImages(appName string) ([]string, error) {
	coll, err := appImagesColl()
	if err != nil {
		return nil, err
	}
	defer coll.Close()
	var img appImages
	err = coll.FindId(appName).One(&img)
	if err != nil {
		if err == mgo.ErrNotFound {
			return []string{}, nil
		}
		return nil, err
	}
	historySize := imageHistorySize()
	if len(img.Images) > historySize {
		img.Images = img.Images[len(img.Images)-historySize:]
	}
	return img.Images, nil
}

func isValidAppImage(appName, imageId string) (bool, error) {
	images, err := listValidAppImages(appName)
	if err != nil && err != mgo.ErrNotFound {
		return false, err
	}
	for _, img := range images {
		if img == imageId {
			return true, nil
		}
	}
	return false, nil
}

func imageHistorySize() int {
	imgHistorySize, _ := config.GetInt("docker:image-history-size")
	if imgHistorySize == 0 {
		imgHistorySize = 10
	}
	return imgHistorySize
}

func deleteAllAppImageNames(appName string) error {
	images, err := listAppImages(appName)
	if err != nil {
		return err
	}
	dataColl, err := imageCustomDataColl()
	if err != nil {
		return err
	}
	defer dataColl.Close()
	_, err = dataColl.RemoveAll(bson.M{"_id": bson.M{"$in": images}})
	if err != nil {
		return err
	}
	coll, err := appImagesColl()
	if err != nil {
		return err
	}
	defer coll.Close()
	return coll.RemoveId(appName)
}

func pullAppImageNames(appName string, images []string) error {
	dataColl, err := imageCustomDataColl()
	if err != nil {
		return err
	}
	defer dataColl.Close()
	_, err = dataColl.RemoveAll(bson.M{"_id": bson.M{"$in": images}})
	if err != nil {
		return err
	}
	coll, err := appImagesColl()
	if err != nil {
		return err
	}
	defer coll.Close()
	return coll.UpdateId(appName, bson.M{"$pullAll": bson.M{"images": images}})
}

func platformImageName(platformName string) string {
	return fmt.Sprintf("%s/%s", basicImageName(), platformName)
}

func basicImageName() string {
	parts := make([]string, 0, 2)
	registry, _ := config.GetString("docker:registry")
	if registry != "" {
		parts = append(parts, registry)
	}
	repoNamespace, _ := config.GetString("docker:repository-namespace")
	parts = append(parts, repoNamespace)
	return strings.Join(parts, "/")
}

func usePlatformImage(app provision.App) bool {
	deploys := app.GetDeploys()
	if (deploys != 0 && deploys%10 == 0) || app.GetUpdatePlatform() {
		return true
	}
	c, err := getOneContainerByAppName(app.GetName())
	if err != nil || c.Image == "" {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package tnt

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestConnect(t *testing.T) {
	assert := assert.New(t)

	box, err := NewBox("", nil)
	assert.NoError(err)
	defer box.Close()

	conn, err := Connect(fmt.Sprintf("127.0.0.1:%d", box.Port), nil)
	if !assert.NoError(err) {
		return
	}
	defer conn.Close()

	assert.Contains(string(conn.Greeting.Version), "Tarantool")
}
<commit_msg>Box.Addr helper<commit_after>package tnt

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestConnect(t *testing.T) {
	assert := assert.New(t)

	box, err := NewBox("", nil)
	assert.NoError(err)
	defer box.Close()

	conn, err := Connect(box.Addr(), nil)
	if !assert.NoError(err) {
		return
	}
	defer conn.Close()

	assert.Contains(string(conn.Greeting.Version), "Tarantool")
}
<|endoftext|>