<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package chartutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"k8s.io/helm/pkg/proto/hapi/chart"
)

const (
	// ChartfileName is the default Chart file name.
	ChartfileName = "Chart.yaml"
	// ValuesfileName is the default values file name.
	ValuesfileName = "values.yaml"
	// TemplatesDir is the relative directory name for templates.
	TemplatesDir = "templates"
	// ChartsDir is the relative directory name for charts dependencies.
	ChartsDir = "charts"
	// IgnorefileName is the name of the Helm ignore file.
	IgnorefileName = ".helmignore"
	// IngressFileName is the name of the example ingress file.
	IngressFileName = "ingress.yaml"
	// DeploymentName is the name of the example deployment file.
	DeploymentName = "deployment.yaml"
	// ServiceName is the name of the example service file.
	ServiceName = "service.yaml"
	// NotesName is the name of the example NOTES.txt file.
	NotesName = "NOTES.txt"
	// HelpersName is the name of the example helpers file.
	HelpersName = "_helpers.tpl"
	// TemplatesTestsDir is the relative directory name for templates tests.
	TemplatesTestsDir = "templates/tests"
	// TestConnectionName is the name of the example connection test file.
	TestConnectionName = "test-connection.yaml"
)

const defaultValues = `# Default values for %s.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

replicaCount: 1

image:
  repository: nginx
  tag: stable
  pullPolicy: IfNotPresent

nameOverride: ""
fullnameOverride: ""

service:
  type: ClusterIP
  port: 80

ingress:
  enabled: false
  annotations: {}
    # kubernetes.io/ingress.class: nginx
    # kubernetes.io/tls-acme: "true"
  paths: []
  hosts:
    - chart-example.local
  tls: []
  #  - secretName: chart-example-tls
  #    hosts:
  #      - chart-example.local

resources: {}
  # We usually recommend not to specify default resources and to leave this as a conscious
  # choice for the user. This also increases chances charts run on environments with little
  # resources, such as Minikube. If you do want to specify resources, uncomment the following
  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
  # limits:
  #   cpu: 100m
  #   memory: 128Mi
  # requests:
  #   cpu: 100m
  #   memory: 128Mi

nodeSelector: {}

tolerations: []

affinity: {}
`

const defaultIgnore = `# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
`

const defaultIngress = `{{- if .Values.ingress.enabled -}}
{{- $fullName := include "<CHARTNAME>.fullname" . -}}
{{- $ingressPaths := .Values.ingress.paths -}}
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: {{ $fullName }}
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
  {{- with .Values.ingress.annotations }}
  annotations:
    {{- toYaml . | nindent 4 }}
  {{- end }}
spec:
{{- if .Values.ingress.tls }}
  tls:
  {{- range .Values.ingress.tls }}
    - hosts:
      {{- range .hosts }}
        - {{ . | quote }}
      {{- end }}
      secretName: {{ .secretName }}
  {{- end }}
{{- end }}
  rules:
  {{- range .Values.ingress.hosts }}
    - host: {{ . | quote }}
      http:
        paths:
	{{- range $ingressPaths }}
          - path: {{ . }}
            backend:
              serviceName: {{ $fullName }}
              servicePort: http
	{{- end }}
  {{- end }}
{{- end }}
`

const defaultDeployment = `apiVersion: apps/v1
kind: Deployment
metadata:
  name: {{ include "<CHARTNAME>.fullname" . }}
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
  replicas: {{ .Values.replicaCount }}
  selector:
    matchLabels:
      app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
      app.kubernetes.io/instance: {{ .Release.Name }}
  template:
    metadata:
      labels:
        app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
        app.kubernetes.io/instance: {{ .Release.Name }}
    spec:
      containers:
        - name: {{ .Chart.Name }}
          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
          imagePullPolicy: {{ .Values.image.pullPolicy }}
          ports:
            - name: http
              containerPort: 80
              protocol: TCP
          livenessProbe:
            httpGet:
              path: /
              port: http
          readinessProbe:
            httpGet:
              path: /
              port: http
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
      {{- with .Values.nodeSelector }}
      nodeSelector:
        {{- toYaml . | nindent 8 }}
      {{- end }}
    {{- with .Values.affinity }}
      affinity:
        {{- toYaml . | nindent 8 }}
    {{- end }}
    {{- with .Values.tolerations }}
      tolerations:
        {{- toYaml . | nindent 8 }}
    {{- end }}
`

const defaultService = `apiVersion: v1
kind: Service
metadata:
  name: {{ include "<CHARTNAME>.fullname" . }}
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
  type: {{ .Values.service.type }}
  ports:
    - port: {{ .Values.service.port }}
      targetPort: http
      protocol: TCP
      name: http
  selector:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
`

const defaultNotes = `1. Get the application URL by running these commands:
{{- if .Values.ingress.enabled }}
{{- range $host := .Values.ingress.hosts }}
  {{- range $.Values.ingress.paths }}
  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host }}{{ . }}
  {{- end }}
{{- end }}
{{- else if contains "NodePort" .Values.service.type }}
  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "<CHARTNAME>.fullname" . }})
  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
  echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
           You can watch the status of by running 'kubectl get svc -w {{ include "<CHARTNAME>.fullname" . }}'
  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "<CHARTNAME>.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
  echo http://$SERVICE_IP:{{ .Values.service.port }}
{{- else if contains "ClusterIP" .Values.service.type }}
  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "<CHARTNAME>.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
  echo "Visit http://127.0.0.1:8080 to use your application"
  kubectl port-forward $POD_NAME 8080:80
{{- end }}
`

const defaultHelpers = `{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "<CHARTNAME>.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "<CHARTNAME>.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}

{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "<CHARTNAME>.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
`

const defaultTestConnection = `apiVersion: v1
kind: Pod
metadata:
  name: "{{ include "<CHARTNAME>.fullname" . }}-test-connection"
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
  annotations:
    "helm.sh/hook": test-success
spec:
  containers:
    - name: wget
      image: busybox
      command: ['wget']
      args:  ['{{ include "<CHARTNAME>.fullname" . }}:{{ .Values.service.port }}']
  restartPolicy: Never
`

// CreateFrom creates a new chart, but scaffolds it from the src chart.
func CreateFrom(chartfile *chart.Metadata, dest string, src string) error {
	schart, err := Load(src)
	if err != nil {
		return fmt.Errorf("could not load %s: %s", src, err)
	}

	schart.Metadata = chartfile

	var updatedTemplates []*chart.Template

	for _, template := range schart.Templates {
		newData := Transform(string(template.Data), "<CHARTNAME>", schart.Metadata.Name)
		updatedTemplates = append(updatedTemplates, &chart.Template{Name: template.Name, Data: newData})
	}

	schart.Templates = updatedTemplates
	if schart.Values != nil {
		schart.Values = &chart.Config{Raw: string(Transform(schart.Values.Raw, "<CHARTNAME>", schart.Metadata.Name))}
	}
	return SaveDir(schart, dest)
}

// Create creates a new chart in a directory.
//
// Inside of dir, this will create a directory based on the name of
// chartfile.Name. It will then write the Chart.yaml into this directory and
// create the (empty) appropriate directories.
//
// The returned string will point to the newly created directory. It will be
// an absolute path, even if the provided base directory was relative.
//
// If dir does not exist, this will return an error.
// If Chart.yaml or any directories cannot be created, this will return an
// error. In such a case, this will attempt to clean up by removing the
// new chart directory.
func Create(chartfile *chart.Metadata, dir string) (string, error) {
	path, err := filepath.Abs(dir)
	if err != nil {
		return path, err
	}

	if fi, err := os.Stat(path); err != nil {
		return path, err
	} else if !fi.IsDir() {
		return path, fmt.Errorf("no such directory %s", path)
	}

	n := chartfile.Name
	cdir := filepath.Join(path, n)
	if fi, err := os.Stat(cdir); err == nil && !fi.IsDir() {
		return cdir, fmt.Errorf("file %s already exists and is not a directory", cdir)
	}
	if err := os.MkdirAll(cdir, 0755); err != nil {
		return cdir, err
	}

	cf := filepath.Join(cdir, ChartfileName)
	if _, err := os.Stat(cf); err != nil {
		if err := SaveChartfile(cf, chartfile); err != nil {
			return cdir, err
		}
	}

	for _, d := range []string{TemplatesDir, TemplatesTestsDir, ChartsDir} {
		if err := os.MkdirAll(filepath.Join(cdir, d), 0755); err != nil {
			return cdir, err
		}
	}

	files := []struct {
		path    string
		content []byte
	}{
		{
			// values.yaml
			path:    filepath.Join(cdir, ValuesfileName),
			content: []byte(fmt.Sprintf(defaultValues, chartfile.Name)),
		},
		{
			// .helmignore
			path:    filepath.Join(cdir, IgnorefileName),
			content: []byte(defaultIgnore),
		},
		{
			// ingress.yaml
			path:    filepath.Join(cdir, TemplatesDir, IngressFileName),
			content: Transform(defaultIngress, "<CHARTNAME>", chartfile.Name),
		},
		{
			// deployment.yaml
			path:    filepath.Join(cdir, TemplatesDir, DeploymentName),
			content: Transform(defaultDeployment, "<CHARTNAME>", chartfile.Name),
		},
		{
			// service.yaml
			path:    filepath.Join(cdir, TemplatesDir, ServiceName),
			content: Transform(defaultService, "<CHARTNAME>", chartfile.Name),
		},
		{
			// NOTES.txt
			path:    filepath.Join(cdir, TemplatesDir, NotesName),
			content: Transform(defaultNotes, "<CHARTNAME>", chartfile.Name),
		},
		{
			// _helpers.tpl
			path:    filepath.Join(cdir, TemplatesDir, HelpersName),
			content: Transform(defaultHelpers, "<CHARTNAME>", chartfile.Name),
		},
		{
			// test-connection.yaml
			path:    filepath.Join(cdir, TemplatesTestsDir, TestConnectionName),
			content: Transform(defaultTestConnection, "<CHARTNAME>", chartfile.Name),
		},
	}

	for _, file := range files {
		if _, err := os.Stat(file.path); err == nil {
			// File exists and is okay. Skip it.
			continue
		}
		if err := ioutil.WriteFile(file.path, file.content, 0644); err != nil {
			return cdir, err
		}
	}
	return cdir, nil
}
<commit_msg>create: include the namespace in NOTES.txt for LoadBalancer status (#5055)<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package chartutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"k8s.io/helm/pkg/proto/hapi/chart"
)

const (
	// ChartfileName is the default Chart file name.
	ChartfileName = "Chart.yaml"
	// ValuesfileName is the default values file name.
	ValuesfileName = "values.yaml"
	// TemplatesDir is the relative directory name for templates.
	TemplatesDir = "templates"
	// ChartsDir is the relative directory name for charts dependencies.
	ChartsDir = "charts"
	// IgnorefileName is the name of the Helm ignore file.
	IgnorefileName = ".helmignore"
	// IngressFileName is the name of the example ingress file.
	IngressFileName = "ingress.yaml"
	// DeploymentName is the name of the example deployment file.
	DeploymentName = "deployment.yaml"
	// ServiceName is the name of the example service file.
	ServiceName = "service.yaml"
	// NotesName is the name of the example NOTES.txt file.
	NotesName = "NOTES.txt"
	// HelpersName is the name of the example helpers file.
	HelpersName = "_helpers.tpl"
	// TemplatesTestsDir is the relative directory name for templates tests.
	TemplatesTestsDir = "templates/tests"
	// TestConnectionName is the name of the example connection test file.
	TestConnectionName = "test-connection.yaml"
)

const defaultValues = `# Default values for %s.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

replicaCount: 1

image:
  repository: nginx
  tag: stable
  pullPolicy: IfNotPresent

nameOverride: ""
fullnameOverride: ""

service:
  type: ClusterIP
  port: 80

ingress:
  enabled: false
  annotations: {}
    # kubernetes.io/ingress.class: nginx
    # kubernetes.io/tls-acme: "true"
  paths: []
  hosts:
    - chart-example.local
  tls: []
  #  - secretName: chart-example-tls
  #    hosts:
  #      - chart-example.local

resources: {}
  # We usually recommend not to specify default resources and to leave this as a conscious
  # choice for the user. This also increases chances charts run on environments with little
  # resources, such as Minikube. If you do want to specify resources, uncomment the following
  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
  # limits:
  #   cpu: 100m
  #   memory: 128Mi
  # requests:
  #   cpu: 100m
  #   memory: 128Mi

nodeSelector: {}

tolerations: []

affinity: {}
`

const defaultIgnore = `# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
`

const defaultIngress = `{{- if .Values.ingress.enabled -}}
{{- $fullName := include "<CHARTNAME>.fullname" . -}}
{{- $ingressPaths := .Values.ingress.paths -}}
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: {{ $fullName }}
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
  {{- with .Values.ingress.annotations }}
  annotations:
    {{- toYaml . | nindent 4 }}
  {{- end }}
spec:
{{- if .Values.ingress.tls }}
  tls:
  {{- range .Values.ingress.tls }}
    - hosts:
      {{- range .hosts }}
        - {{ . | quote }}
      {{- end }}
      secretName: {{ .secretName }}
  {{- end }}
{{- end }}
  rules:
  {{- range .Values.ingress.hosts }}
    - host: {{ . | quote }}
      http:
        paths:
	{{- range $ingressPaths }}
          - path: {{ . }}
            backend:
              serviceName: {{ $fullName }}
              servicePort: http
	{{- end }}
  {{- end }}
{{- end }}
`

const defaultDeployment = `apiVersion: apps/v1
kind: Deployment
metadata:
  name: {{ include "<CHARTNAME>.fullname" . }}
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
  replicas: {{ .Values.replicaCount }}
  selector:
    matchLabels:
      app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
      app.kubernetes.io/instance: {{ .Release.Name }}
  template:
    metadata:
      labels:
        app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
        app.kubernetes.io/instance: {{ .Release.Name }}
    spec:
      containers:
        - name: {{ .Chart.Name }}
          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
          imagePullPolicy: {{ .Values.image.pullPolicy }}
          ports:
            - name: http
              containerPort: 80
              protocol: TCP
          livenessProbe:
            httpGet:
              path: /
              port: http
          readinessProbe:
            httpGet:
              path: /
              port: http
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
      {{- with .Values.nodeSelector }}
      nodeSelector:
        {{- toYaml . | nindent 8 }}
      {{- end }}
    {{- with .Values.affinity }}
      affinity:
        {{- toYaml . | nindent 8 }}
    {{- end }}
    {{- with .Values.tolerations }}
      tolerations:
        {{- toYaml . | nindent 8 }}
    {{- end }}
`

const defaultService = `apiVersion: v1
kind: Service
metadata:
  name: {{ include "<CHARTNAME>.fullname" . }}
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
  type: {{ .Values.service.type }}
  ports:
    - port: {{ .Values.service.port }}
      targetPort: http
      protocol: TCP
      name: http
  selector:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
`

const defaultNotes = `1. Get the application URL by running these commands:
{{- if .Values.ingress.enabled }}
{{- range $host := .Values.ingress.hosts }}
  {{- range $.Values.ingress.paths }}
  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host }}{{ . }}
  {{- end }}
{{- end }}
{{- else if contains "NodePort" .Values.service.type }}
  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "<CHARTNAME>.fullname" . }})
  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
  echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "<CHARTNAME>.fullname" . }}'
  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "<CHARTNAME>.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
  echo http://$SERVICE_IP:{{ .Values.service.port }}
{{- else if contains "ClusterIP" .Values.service.type }}
  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "<CHARTNAME>.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
  echo "Visit http://127.0.0.1:8080 to use your application"
  kubectl port-forward $POD_NAME 8080:80
{{- end }}
`

const defaultHelpers = `{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "<CHARTNAME>.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "<CHARTNAME>.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}

{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "<CHARTNAME>.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
`

const defaultTestConnection = `apiVersion: v1
kind: Pod
metadata:
  name: "{{ include "<CHARTNAME>.fullname" . }}-test-connection"
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
  annotations:
    "helm.sh/hook": test-success
spec:
  containers:
    - name: wget
      image: busybox
      command: ['wget']
      args:  ['{{ include "<CHARTNAME>.fullname" . }}:{{ .Values.service.port }}']
  restartPolicy: Never
`

// CreateFrom creates a new chart, but scaffolds it from the src chart.
func CreateFrom(chartfile *chart.Metadata, dest string, src string) error {
	schart, err := Load(src)
	if err != nil {
		return fmt.Errorf("could not load %s: %s", src, err)
	}

	schart.Metadata = chartfile

	var updatedTemplates []*chart.Template

	for _, template := range schart.Templates {
		newData := Transform(string(template.Data), "<CHARTNAME>", schart.Metadata.Name)
		updatedTemplates = append(updatedTemplates, &chart.Template{Name: template.Name, Data: newData})
	}

	schart.Templates = updatedTemplates
	if schart.Values != nil {
		schart.Values = &chart.Config{Raw: string(Transform(schart.Values.Raw, "<CHARTNAME>", schart.Metadata.Name))}
	}
	return SaveDir(schart, dest)
}

// Create creates a new chart in a directory.
//
// Inside of dir, this will create a directory based on the name of
// chartfile.Name. It will then write the Chart.yaml into this directory and
// create the (empty) appropriate directories.
//
// The returned string will point to the newly created directory. It will be
// an absolute path, even if the provided base directory was relative.
//
// If dir does not exist, this will return an error.
// If Chart.yaml or any directories cannot be created, this will return an
// error. In such a case, this will attempt to clean up by removing the
// new chart directory.
func Create(chartfile *chart.Metadata, dir string) (string, error) {
	path, err := filepath.Abs(dir)
	if err != nil {
		return path, err
	}

	if fi, err := os.Stat(path); err != nil {
		return path, err
	} else if !fi.IsDir() {
		return path, fmt.Errorf("no such directory %s", path)
	}

	n := chartfile.Name
	cdir := filepath.Join(path, n)
	if fi, err := os.Stat(cdir); err == nil && !fi.IsDir() {
		return cdir, fmt.Errorf("file %s already exists and is not a directory", cdir)
	}
	if err := os.MkdirAll(cdir, 0755); err != nil {
		return cdir, err
	}

	cf := filepath.Join(cdir, ChartfileName)
	if _, err := os.Stat(cf); err != nil {
		if err := SaveChartfile(cf, chartfile); err != nil {
			return cdir, err
		}
	}

	for _, d := range []string{TemplatesDir, TemplatesTestsDir, ChartsDir} {
		if err := os.MkdirAll(filepath.Join(cdir, d), 0755); err != nil {
			return cdir, err
		}
	}

	files := []struct {
		path    string
		content []byte
	}{
		{
			// values.yaml
			path:    filepath.Join(cdir, ValuesfileName),
			content: []byte(fmt.Sprintf(defaultValues, chartfile.Name)),
		},
		{
			// .helmignore
			path:    filepath.Join(cdir, IgnorefileName),
			content: []byte(defaultIgnore),
		},
		{
			// ingress.yaml
			path:    filepath.Join(cdir, TemplatesDir, IngressFileName),
			content: Transform(defaultIngress, "<CHARTNAME>", chartfile.Name),
		},
		{
			// deployment.yaml
			path:    filepath.Join(cdir, TemplatesDir, DeploymentName),
			content: Transform(defaultDeployment, "<CHARTNAME>", chartfile.Name),
		},
		{
			// service.yaml
			path:    filepath.Join(cdir, TemplatesDir, ServiceName),
			content: Transform(defaultService, "<CHARTNAME>", chartfile.Name),
		},
		{
			// NOTES.txt
			path:    filepath.Join(cdir, TemplatesDir, NotesName),
			content: Transform(defaultNotes, "<CHARTNAME>", chartfile.Name),
		},
		{
			// _helpers.tpl
			path:    filepath.Join(cdir, TemplatesDir, HelpersName),
			content: Transform(defaultHelpers, "<CHARTNAME>", chartfile.Name),
		},
		{
			// test-connection.yaml
			path:    filepath.Join(cdir, TemplatesTestsDir, TestConnectionName),
			content: Transform(defaultTestConnection, "<CHARTNAME>", chartfile.Name),
		},
	}

	for _, file := range files {
		if _, err := os.Stat(file.path); err == nil {
			// File exists and is okay. Skip it.
			continue
		}
		if err := ioutil.WriteFile(file.path, file.content, 0644); err != nil {
			return cdir, err
		}
	}
	return cdir, nil
}
<|endoftext|>


<commit_before>package tk

import (
	"src.elv.sh/pkg/cli/term"
	"src.elv.sh/pkg/ui"
)

// VScrollbarContainer is a Renderer consisting of content and a vertical
// scrollbar on the right.
type VScrollbarContainer struct {
	Content   Renderer
	Scrollbar VScrollbar
}

func (v VScrollbarContainer) Render(width, height int) *term.Buffer {
	buf := v.Content.Render(width-1, height)
	buf.ExtendRight(v.Scrollbar.Render(1, height))
	return buf
}

// VScrollbar is a Renderer for a vertical scrollbar.
type VScrollbar struct {
	Total int
	Low   int
	High  int
}

var (
	vscrollbarThumb  = ui.T(" ", ui.FgMagenta, ui.Inverse)
	vscrollbarTrough = ui.T("│", ui.FgMagenta)
)

func (v VScrollbar) Render(width, height int) *term.Buffer {
	posLow, posHigh := findScrollInterval(v.Total, v.Low, v.High, height)
	bb := term.NewBufferBuilder(1)
	for i := 0; i < height; i++ {
		if i > 0 {
			bb.Newline()
		}
		if posLow <= i && i < posHigh {
			bb.WriteStyled(vscrollbarThumb)
		} else {
			bb.WriteStyled(vscrollbarTrough)
		}
	}
	return bb.Buffer()
}

// HScrollbar is a Renderer for a horizontal scrollbar.
type HScrollbar struct {
	Total int
	Low   int
	High  int
}

var (
	hscrollbarThumb  = ui.T(" ", ui.FgMagenta, ui.Inverse)
	hscrollbarTrough = ui.T("━", ui.FgMagenta)
)

func (h HScrollbar) Render(width, height int) *term.Buffer {
	posLow, posHigh := findScrollInterval(h.Total, h.Low, h.High, width)
	bb := term.NewBufferBuilder(width)
	for i := 0; i < width; i++ {
		if posLow <= i && i < posHigh {
			bb.WriteStyled(hscrollbarThumb)
		} else {
			bb.WriteStyled(hscrollbarTrough)
		}
	}
	return bb.Buffer()
}

func findScrollInterval(n, low, high, height int) (int, int) {
	f := func(i int) int {
		return int(float64(i)/float64(n)*float64(height) + 0.5)
	}
	scrollLow, scrollHigh := f(low), f(high)
	if scrollLow == scrollHigh {
		if scrollHigh == height {
			scrollLow--
		} else {
			scrollHigh++
		}
	}
	return scrollLow, scrollHigh
}
<commit_msg>pkg/cli/tk: Tweak the scrollbar algorithm.<commit_after>package tk

import (
	"src.elv.sh/pkg/cli/term"
	"src.elv.sh/pkg/ui"
)

// VScrollbarContainer is a Renderer consisting of content and a vertical
// scrollbar on the right.
type VScrollbarContainer struct {
	Content   Renderer
	Scrollbar VScrollbar
}

func (v VScrollbarContainer) Render(width, height int) *term.Buffer {
	buf := v.Content.Render(width-1, height)
	buf.ExtendRight(v.Scrollbar.Render(1, height))
	return buf
}

// VScrollbar is a Renderer for a vertical scrollbar.
type VScrollbar struct {
	Total int
	Low   int
	High  int
}

var (
	vscrollbarThumb  = ui.T(" ", ui.FgMagenta, ui.Inverse)
	vscrollbarTrough = ui.T("│", ui.FgMagenta)
)

func (v VScrollbar) Render(width, height int) *term.Buffer {
	posLow, posHigh := findScrollInterval(v.Total, v.Low, v.High, height)
	bb := term.NewBufferBuilder(1)
	for i := 0; i < height; i++ {
		if i > 0 {
			bb.Newline()
		}
		if posLow <= i && i < posHigh {
			bb.WriteStyled(vscrollbarThumb)
		} else {
			bb.WriteStyled(vscrollbarTrough)
		}
	}
	return bb.Buffer()
}

// HScrollbar is a Renderer for a horizontal scrollbar.
type HScrollbar struct {
	Total int
	Low   int
	High  int
}

var (
	hscrollbarThumb  = ui.T(" ", ui.FgMagenta, ui.Inverse)
	hscrollbarTrough = ui.T("━", ui.FgMagenta)
)

func (h HScrollbar) Render(width, height int) *term.Buffer {
	posLow, posHigh := findScrollInterval(h.Total, h.Low, h.High, width)
	bb := term.NewBufferBuilder(width)
	for i := 0; i < width; i++ {
		if posLow <= i && i < posHigh {
			bb.WriteStyled(hscrollbarThumb)
		} else {
			bb.WriteStyled(hscrollbarTrough)
		}
	}
	return bb.Buffer()
}

func findScrollInterval(n, low, high, height int) (int, int) {
	f := func(i int) int {
		return int(float64(i)/float64(n)*float64(height) + 0.5)
	}
	scrollLow := f(low)
	// We use the following instead of f(high), so that the size of the
	// scrollbar remains the same as long as the window size remains the same.
	scrollHigh := scrollLow + f(high-low)

	if scrollLow == scrollHigh {
		if scrollHigh == height {
			scrollLow--
		} else {
			scrollHigh++
		}
	}
	return scrollLow, scrollHigh
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"fmt"
	"io"
	"io/ioutil"

	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"k8s.io/apimachinery/pkg/util/sets"
	genericapiserver "k8s.io/apiserver/pkg/server"
	"k8s.io/apiserver/pkg/server/filters"
	genericoptions "k8s.io/apiserver/pkg/server/options"
	kubeclientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1alpha1"
	"k8s.io/kube-aggregator/pkg/apiserver"
)

const defaultEtcdPathPrefix = "/registry/kube-aggregator.kubernetes.io/"

type AggregatorOptions struct {
	RecommendedOptions *genericoptions.RecommendedOptions

	// ProxyClientCert/Key are the client cert used to identify this proxy. Backing APIServices use
	// this to confirm the proxy's identity
	ProxyClientCertFile string
	ProxyClientKeyFile  string

	// CoreAPIKubeconfig is a filename for a kubeconfig file to contact the core API server wtih
	// If it is not set, the in cluster config is used
	CoreAPIKubeconfig string

	StdOut io.Writer
	StdErr io.Writer
}

// NewCommandStartAggregator provides a CLI handler for 'start master' command
func NewCommandStartAggregator(out, err io.Writer, stopCh <-chan struct{}) *cobra.Command {
	o := NewDefaultOptions(out, err)

	cmd := &cobra.Command{
		Short: "Launch a API aggregator and proxy server",
		Long:  "Launch a API aggregator and proxy server",
		RunE: func(c *cobra.Command, args []string) error {
			if err := o.Complete(); err != nil {
				return err
			}
			if err := o.Validate(args); err != nil {
				return err
			}
			if err := o.RunAggregator(stopCh); err != nil {
				return err
			}
			return nil
		},
	}

	o.AddFlags(cmd.Flags())
	return cmd
}

// AddFlags is necessary because hyperkube doesn't work using cobra, so we have to have different registration and execution paths
func (o *AggregatorOptions) AddFlags(fs *pflag.FlagSet) {
	o.RecommendedOptions.AddFlags(fs)
	fs.StringVar(&o.ProxyClientCertFile, "proxy-client-cert-file", o.ProxyClientCertFile, "client certificate used identify the proxy to the API server")
	fs.StringVar(&o.ProxyClientKeyFile, "proxy-client-key-file", o.ProxyClientKeyFile, "client certificate key used identify the proxy to the API server")
	fs.StringVar(&o.CoreAPIKubeconfig, "core-kubeconfig", o.CoreAPIKubeconfig, ""+
		"kubeconfig file pointing at the 'core' kubernetes server with enough rights to get,list,watch "+
		" services,endpoints.  If not set, the in-cluster config is used")
}

// NewDefaultOptions builds a "normal" set of options.  You wouldn't normally expose this, but hyperkube isn't cobra compatible
func NewDefaultOptions(out, err io.Writer) *AggregatorOptions {
	o := &AggregatorOptions{
		RecommendedOptions: genericoptions.NewRecommendedOptions(defaultEtcdPathPrefix, apiserver.Scheme, apiserver.Codecs.LegacyCodec(v1alpha1.SchemeGroupVersion)),

		StdOut: out,
		StdErr: err,
	}
	o.RecommendedOptions.SecureServing.ServingOptions.BindPort = 443
	return o
}

func (o AggregatorOptions) Validate(args []string) error {
	return nil
}

func (o *AggregatorOptions) Complete() error {
	return nil
}

func (o AggregatorOptions) RunAggregator(stopCh <-chan struct{}) error {
	// TODO have a "real" external address
	if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, nil); err != nil {
		return fmt.Errorf("error creating self-signed certificates: %v", err)
	}

	serverConfig := genericapiserver.NewConfig().
		WithSerializer(apiserver.Codecs)

	if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
		return err
	}
	serverConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
		sets.NewString("watch", "proxy"),
		sets.NewString("attach", "exec", "proxy", "log", "portforward"),
	)

	var kubeconfig *rest.Config
	var err error
	if len(o.CoreAPIKubeconfig) > 0 {
		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.CoreAPIKubeconfig}
		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})

		kubeconfig, err = loader.ClientConfig()

	} else {
		kubeconfig, err = rest.InClusterConfig()
	}
	if err != nil {
		return err
	}

	coreAPIServerClient, err := kubeclientset.NewForConfig(kubeconfig)
	if err != nil {
		return err
	}

	config := apiserver.Config{
		GenericConfig:       serverConfig,
		CoreAPIServerClient: coreAPIServerClient,
	}

	config.ProxyClientCert, err = ioutil.ReadFile(o.ProxyClientCertFile)
	if err != nil {
		return err
	}
	config.ProxyClientKey, err = ioutil.ReadFile(o.ProxyClientKeyFile)
	if err != nil {
		return err
	}

	server, err := config.Complete().NewWithDelegate(genericapiserver.EmptyDelegate, stopCh)
	if err != nil {
		return err
	}
	return server.GenericAPIServer.PrepareRun().Run(stopCh)
}
<commit_msg>require codecfactory<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"fmt"
	"io"
	"io/ioutil"

	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"k8s.io/apimachinery/pkg/util/sets"
	genericapiserver "k8s.io/apiserver/pkg/server"
	"k8s.io/apiserver/pkg/server/filters"
	genericoptions "k8s.io/apiserver/pkg/server/options"
	kubeclientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1alpha1"
	"k8s.io/kube-aggregator/pkg/apiserver"
)

const defaultEtcdPathPrefix = "/registry/kube-aggregator.kubernetes.io/"

type AggregatorOptions struct {
	RecommendedOptions *genericoptions.RecommendedOptions

	// ProxyClientCert/Key are the client cert used to identify this proxy. Backing APIServices use
	// this to confirm the proxy's identity
	ProxyClientCertFile string
	ProxyClientKeyFile  string

	// CoreAPIKubeconfig is a filename for a kubeconfig file to contact the core API server wtih
	// If it is not set, the in cluster config is used
	CoreAPIKubeconfig string

	StdOut io.Writer
	StdErr io.Writer
}

// NewCommandStartAggregator provides a CLI handler for 'start master' command
func NewCommandStartAggregator(out, err io.Writer, stopCh <-chan struct{}) *cobra.Command {
	o := NewDefaultOptions(out, err)

	cmd := &cobra.Command{
		Short: "Launch a API aggregator and proxy server",
		Long:  "Launch a API aggregator and proxy server",
		RunE: func(c *cobra.Command, args []string) error {
			if err := o.Complete(); err != nil {
				return err
			}
			if err := o.Validate(args); err != nil {
				return err
			}
			if err := o.RunAggregator(stopCh); err != nil {
				return err
			}
			return nil
		},
	}

	o.AddFlags(cmd.Flags())
	return cmd
}

// AddFlags is necessary because hyperkube doesn't work using cobra, so we have to have different registration and execution paths
func (o *AggregatorOptions) AddFlags(fs *pflag.FlagSet) {
	o.RecommendedOptions.AddFlags(fs)
	fs.StringVar(&o.ProxyClientCertFile, "proxy-client-cert-file", o.ProxyClientCertFile, "client certificate used identify the proxy to the API server")
	fs.StringVar(&o.ProxyClientKeyFile, "proxy-client-key-file", o.ProxyClientKeyFile, "client certificate key used identify the proxy to the API server")
	fs.StringVar(&o.CoreAPIKubeconfig, "core-kubeconfig", o.CoreAPIKubeconfig, ""+
		"kubeconfig file pointing at the 'core' kubernetes server with enough rights to get,list,watch "+
		" services,endpoints.  If not set, the in-cluster config is used")
}

// NewDefaultOptions builds a "normal" set of options.  You wouldn't normally expose this, but hyperkube isn't cobra compatible
func NewDefaultOptions(out, err io.Writer) *AggregatorOptions {
	o := &AggregatorOptions{
		RecommendedOptions: genericoptions.NewRecommendedOptions(defaultEtcdPathPrefix, apiserver.Scheme, apiserver.Codecs.LegacyCodec(v1alpha1.SchemeGroupVersion)),

		StdOut: out,
		StdErr: err,
	}
	o.RecommendedOptions.SecureServing.ServingOptions.BindPort = 443
	return o
}

func (o AggregatorOptions) Validate(args []string) error {
	return nil
}

func (o *AggregatorOptions) Complete() error {
	return nil
}

func (o AggregatorOptions) RunAggregator(stopCh <-chan struct{}) error {
	// TODO have a "real" external address
	if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, nil); err != nil {
		return fmt.Errorf("error creating self-signed certificates: %v", err)
	}

	serverConfig := genericapiserver.NewConfig(apiserver.Codecs)

	if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
		return err
	}
	serverConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
		sets.NewString("watch", "proxy"),
		sets.NewString("attach", "exec", "proxy", "log", "portforward"),
	)

	var kubeconfig *rest.Config
	var err error
	if len(o.CoreAPIKubeconfig) > 0 {
		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.CoreAPIKubeconfig}
		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})

		kubeconfig, err = loader.ClientConfig()

	} else {
		kubeconfig, err = rest.InClusterConfig()
	}
	if err != nil {
		return err
	}

	coreAPIServerClient, err := kubeclientset.NewForConfig(kubeconfig)
	if err != nil {
		return err
	}

	config := apiserver.Config{
		GenericConfig:       serverConfig,
		CoreAPIServerClient: coreAPIServerClient,
	}

	config.ProxyClientCert, err = ioutil.ReadFile(o.ProxyClientCertFile)
	if err != nil {
		return err
	}
	config.ProxyClientKey, err = ioutil.ReadFile(o.ProxyClientKeyFile)
	if err != nil {
		return err
	}

	server, err := config.Complete().NewWithDelegate(genericapiserver.EmptyDelegate, stopCh)
	if err != nil {
		return err
	}
	return server.GenericAPIServer.PrepareRun().Run(stopCh)
}
<|endoftext|>


<commit_before>package distributor

import (
	"math"
	"net/http"

	"github.com/weaveworks/common/httpgrpc"

	"github.com/cortexproject/cortex/pkg/util"

	"github.com/grafana/loki/pkg/loghttp"
	"github.com/grafana/loki/pkg/logproto"
	"github.com/grafana/loki/pkg/logql/unmarshal"
	unmarshal_legacy "github.com/grafana/loki/pkg/logql/unmarshal/legacy"
)

var contentType = http.CanonicalHeaderKey("Content-Type")

const applicationJSON = "application/json"

// PushHandler reads a snappy-compressed proto from the HTTP body.
func (d *Distributor) PushHandler(w http.ResponseWriter, r *http.Request) {
	req, err := ParseRequest(r)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	_, err = d.Push(r.Context(), req)
	if err == nil {
		w.WriteHeader(http.StatusNoContent)
		return
	}

	resp, ok := httpgrpc.HTTPResponseFromError(err)
	if ok {
		http.Error(w, string(resp.Body), int(resp.Code))
	} else {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func ParseRequest(r *http.Request) (*logproto.PushRequest, error) {
	var req logproto.PushRequest

	switch r.Header.Get(contentType) {
	case applicationJSON:
		var err error

		// todo once https://github.com/weaveworks/common/commit/73225442af7da93ec8f6a6e2f7c8aafaee3f8840 is in Loki.
		// We can try to pass the body as bytes.buffer instead to avoid reading into another buffer.
		if loghttp.GetVersion(r.RequestURI) == loghttp.VersionV1 {
			err = unmarshal.DecodePushRequest(r.Body, &req)
		} else {
			err = unmarshal_legacy.DecodePushRequest(r.Body, &req)
		}

		if err != nil {
			return nil, err
		}

	default:
		if err := util.ParseProtoReader(r.Context(), r.Body, int(r.ContentLength), math.MaxInt32, &req, util.RawSnappy); err != nil {
			return nil, err
		}
	}
	return &req, nil
}
<commit_msg>Re-introduce https://github.com/grafana/loki/pull/3178. (#3233)<commit_after>package distributor

import (
	"math"
	"net/http"

	"github.com/cortexproject/cortex/pkg/util"
	"github.com/dustin/go-humanize"
	"github.com/go-kit/kit/log/level"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"github.com/weaveworks/common/httpgrpc"
	"github.com/weaveworks/common/user"

	"github.com/grafana/loki/pkg/loghttp"
	"github.com/grafana/loki/pkg/logproto"
	"github.com/grafana/loki/pkg/logql/unmarshal"
	unmarshal_legacy "github.com/grafana/loki/pkg/logql/unmarshal/legacy"
	lokiutil "github.com/grafana/loki/pkg/util"
)

var (
	contentType = http.CanonicalHeaderKey("Content-Type")

	bytesIngested = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: "loki",
		Name:      "distributor_bytes_received_total",
		Help:      "The total number of uncompressed bytes received per tenant",
	}, []string{"tenant"})
	linesIngested = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: "loki",
		Name:      "distributor_lines_received_total",
		Help:      "The total number of lines received per tenant",
	}, []string{"tenant"})
)

const applicationJSON = "application/json"

// PushHandler reads a snappy-compressed proto from the HTTP body.
func (d *Distributor) PushHandler(w http.ResponseWriter, r *http.Request) {
	req, err := ParseRequest(r)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	_, err = d.Push(r.Context(), req)
	if err == nil {
		w.WriteHeader(http.StatusNoContent)
		return
	}

	resp, ok := httpgrpc.HTTPResponseFromError(err)
	if ok {
		http.Error(w, string(resp.Body), int(resp.Code))
	} else {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func ParseRequest(r *http.Request) (*logproto.PushRequest, error) {
	userID, _ := user.ExtractOrgID(r.Context())
	logger := util.WithContext(r.Context(), util.Logger)
	body := lokiutil.NewSizeReader(r.Body)
	contentType := r.Header.Get(contentType)
	var req logproto.PushRequest

	defer func() {
		var (
			entriesSize      int64
			streamLabelsSize int64
			totalEntries     int64
		)

		for _, s := range req.Streams {
			streamLabelsSize += int64(len(s.Labels))
			for _, e := range s.Entries {
				totalEntries++
				entriesSize += int64(len(e.Line))
			}
		}

		// incrementing tenant metrics if we have a tenant.
		if totalEntries != 0 && userID != "" {
			bytesIngested.WithLabelValues(userID).Add(float64(entriesSize))
			linesIngested.WithLabelValues(userID).Add(float64(totalEntries))
		}

		level.Debug(logger).Log(
			"msg", "push request parsed",
			"path", r.URL.Path,
			"contentType", contentType,
			"bodySize", humanize.Bytes(uint64(body.Size())),
			"streams", len(req.Streams),
			"entries", totalEntries,
			"streamLabelsSize", humanize.Bytes(uint64(streamLabelsSize)),
			"entriesSize", humanize.Bytes(uint64(entriesSize)),
			"totalSize", humanize.Bytes(uint64(entriesSize+streamLabelsSize)),
		)
	}()

	switch r.Header.Get(contentType) {
	case applicationJSON:
		var err error

		// todo once https://github.com/weaveworks/common/commit/73225442af7da93ec8f6a6e2f7c8aafaee3f8840 is in Loki.
		// We can try to pass the body as bytes.buffer instead to avoid reading into another buffer.
		if loghttp.GetVersion(r.RequestURI) == loghttp.VersionV1 {
			err = unmarshal.DecodePushRequest(r.Body, &req)
		} else {
			err = unmarshal_legacy.DecodePushRequest(r.Body, &req)
		}

		if err != nil {
			return nil, err
		}

	default:
		if err := util.ParseProtoReader(r.Context(), r.Body, int(r.ContentLength), math.MaxInt32, &req, util.RawSnappy); err != nil {
			return nil, err
		}
	}
	return &req, nil
}
<|endoftext|>


<commit_before>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package xds

import (
	"context"
	"errors"
	"io"
	"reflect"
	"strconv"
	"sync/atomic"
	"time"

	envoy_api_v2 "github.com/cilium/cilium/pkg/envoy/envoy/api/v2"
	"github.com/cilium/cilium/pkg/logging/logfields"

	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes/any"
	"github.com/sirupsen/logrus"
)

const (
	// AnyTypeURL is the default type URL to use for ADS resource sets.
	AnyTypeURL = ""
)

var (
	// ErrNoADSTypeURL is the error returned when receiving a request without
	// a type URL from an ADS stream.
	ErrNoADSTypeURL = errors.New("type URL is required for ADS")

	// ErrUnknownTypeURL is the error returned when receiving a request with
	// an unknown type URL.
	ErrUnknownTypeURL = errors.New("unknown type URL")

	// ErrInvalidVersionInfo is the error returned when receiving a request
	// with a version info that is not a positive integer.
	ErrInvalidVersionInfo = errors.New("invalid version info")

	// ErrResourceWatch is the error returned whenever an internal error
	// occurs while waiting for new versions of resources.
	ErrResourceWatch = errors.New("resource watch failed")
)

// Server implements the handling of xDS streams.
type Server struct {
	// watchers maps each supported type URL to its corresponding resource
	// watcher.
	watchers map[string]*ResourceWatcher

	// ackObservers maps each supported type URL to its corresponding observer
	// of ACKs received from Envoy nodes.
	ackObservers map[string]ResourceVersionAckObserver

	// lastStreamID is the identifier of the last processed stream.
	// It is incremented atomically when starting the handling of a new stream.
	lastStreamID uint64
}

// ResourceTypeConfiguration is the configuration of the XDS server for a
// resource type.
type ResourceTypeConfiguration struct {
	// Source contains the resources of this type.
	Source ObservableResourceSource

	// AckObserver is called back whenever a node acknowledges having applied a
	// version of the resources of this type.
	AckObserver ResourceVersionAckObserver
}

// NewServer creates an xDS gRPC stream handler using the given resource
// sources.
// types maps each supported resource type URL to its corresponding resource
// source and ACK observer.
func NewServer(resourceTypes map[string]*ResourceTypeConfiguration,
	resourceAccessTimeout time.Duration) *Server {
	watchers := make(map[string]*ResourceWatcher, len(resourceTypes))
	ackObservers := make(map[string]ResourceVersionAckObserver, len(resourceTypes))
	for typeURL, resType := range resourceTypes {
		w := NewResourceWatcher(typeURL, resType.Source, resourceAccessTimeout)
		resType.Source.AddResourceVersionObserver(w)
		watchers[typeURL] = w

		if resType.AckObserver != nil {
			ackObservers[typeURL] = resType.AckObserver
		}
	}

	// TODO: Unregister the watchers when stopping the server.

	return &Server{watchers: watchers, ackObservers: ackObservers}
}

func getXDSRequestFields(req *envoy_api_v2.DiscoveryRequest) logrus.Fields {
	return logrus.Fields{
		logfields.XDSVersionInfo: req.GetVersionInfo(),
		logfields.XDSClientNode:  req.GetNode(),
		logfields.XDSTypeURL:     req.GetTypeUrl(),
		logfields.XDSNonce:       req.GetResponseNonce(),
	}
}

// HandleRequestStream receives and processes the requests from an xDS stream.
func (s *Server) HandleRequestStream(ctx context.Context, stream Stream, defaultTypeURL string) error {
	// increment stream count
	streamID := atomic.AddUint64(&s.lastStreamID, 1)

	streamLog := log.WithField(logfields.XDSStreamID, streamID)

	reqCh := make(chan *envoy_api_v2.DiscoveryRequest)

	stopRecv := make(chan struct{})
	defer close(stopRecv)

	go func() {
		defer close(reqCh)
		for {
			req, err := stream.Recv()
			if err != nil {
				if err == io.EOF {
					streamLog.Debug("xDS stream closed")
				} else {
					streamLog.WithError(err).Error("error while receiving request from xDS stream")
				}
				return
			}
			if req == nil {
				streamLog.Error("received nil request from xDS stream; stopping xDS stream handling")
				return
			}
			if req.GetTypeUrl() == "" {
				req.TypeUrl = defaultTypeURL
			}
			streamLog.WithFields(getXDSRequestFields(req)).Debug("received request from xDS stream")
			select {
			case <-stopRecv:
				streamLog.Debug("stopping xDS stream handling")
				return
			case reqCh <- req:
			}
		}
	}()

	return s.processRequestStream(ctx, streamLog, stream, reqCh, defaultTypeURL)
}

// perTypeStreamState is the state maintained per resource type for each
// xDS stream.
type perTypeStreamState struct {
	// typeURL identifies the resource type.
	typeURL string

	// pendingWatchCancel is a pending watch on this resource type.
	// If nil, no watch is pending.
	pendingWatchCancel context.CancelFunc

	// nonce is the nonce sent in the last response to a request for this
	// resource type.
	nonce string

	// version is the last version sent. This is needed so that we'll know
	// if a new request is an ACK (VersionInfo matches current version), or a NACK
	// (VersionInfo matches an earlier version).
	version uint64

	// resourceNames is the list of names of resources sent in the last
	// response to a request for this resource type.
	resourceNames []string
}

// processRequestStream processes the requests in an xDS stream from a channel.
func (s *Server) processRequestStream(ctx context.Context, streamLog *logrus.Entry, stream Stream,
	reqCh <-chan *envoy_api_v2.DiscoveryRequest, defaultTypeURL string) error {
	// The request state for every type URL.
	typeStates := make([]perTypeStreamState, len(s.watchers))
	defer func() {
		for _, state := range typeStates {
			if state.pendingWatchCancel != nil {
				state.pendingWatchCancel()
			}
		}
	}()

	// A map of a resource type's URL to the corresponding index in typeStates
	// for the resource type.
	typeIndexes := make(map[string]int, len(typeStates))

	// The set of channels to select from. Since the set of channels is
	// dynamic, we use reflection for selection.
	// The indexes in selectCases from 0 to len(typeStates)-1 match the indexes
	// in typeStates.
	selectCases := make([]reflect.SelectCase, len(typeStates)+2)

	// The last select case index is always the request channel.
	reqChIndex := len(selectCases) - 1
	selectCases[reqChIndex] = reflect.SelectCase{
		Dir:  reflect.SelectRecv,
		Chan: reflect.ValueOf(reqCh),
	}

	// The next-to-last select case is the context's Done channel.
	doneChIndex := reqChIndex - 1
	selectCases[doneChIndex] = reflect.SelectCase{
		Dir:  reflect.SelectRecv,
		Chan: reflect.ValueOf(ctx.Done()),
	}

	// Initially there are no pending watches, so just select a dead channel
	// that will never be selected.
	quietCh := make(chan *VersionedResources)
	defer close(quietCh)
	quietChValue := reflect.ValueOf(quietCh)

	i := 0
	for typeURL := range s.watchers {
		typeStates[i] = perTypeStreamState{
			typeURL: typeURL,
		}

		selectCases[i] = reflect.SelectCase{
			Dir:  reflect.SelectRecv,
			Chan: quietChValue,
		}

		typeIndexes[typeURL] = i

		i++
	}

	streamLog.Info("starting xDS stream processing")

	// The last nonce returned in a response in this stream.
	var responseNonce uint64

	for {
		// Process either a new request from the xDS stream or a response
		// from the resource watcher.
		chosen, recv, recvOK := reflect.Select(selectCases)

		switch chosen {
		case doneChIndex: // Context got canceled.
			streamLog.WithError(ctx.Err()).Error("xDS stream context canceled")
			return ctx.Err()

		case reqChIndex: // Request received from the stream.
			if !recvOK {
				streamLog.Info("xDS stream closed")
				return nil
			}

			req := recv.Interface().(*envoy_api_v2.DiscoveryRequest)

			requestLog := streamLog.WithFields(getXDSRequestFields(req))

			// Ensure that the version info is a string that was sent by this
			// server or the empty string (the first request in a stream should
			// always have an empty version info).
			var versionInfo *uint64
			if req.GetVersionInfo() != "" {
				versionInfoVal, err := strconv.ParseUint(req.VersionInfo, 10, 64)
				if err != nil {
					requestLog.Errorf("invalid version info in xDS request, not a uint64")
					return ErrInvalidVersionInfo
				}
				versionInfo = &versionInfoVal
			}

			typeURL := req.GetTypeUrl()
			if defaultTypeURL == AnyTypeURL && typeURL == "" {
				requestLog.Error("no type URL given in ADS request")
				return ErrNoADSTypeURL
			}

			index, exists := typeIndexes[typeURL]

			if !exists {
				requestLog.Error("unknown type URL in xDS request")
				return ErrUnknownTypeURL
			}

			state := &typeStates[index]

			// Only handle a request if the nonce is valid, i.e. if it's
			// the first request (which nonce is "") or if it's the nonce of
			// the last response that was sent.
			//
			// If the nonce is different (stale), the client hasn't processed
			// the last response yet. Ignore every request until it processes
			// that response and sends a request with that response's nonce.
			if state.nonce == "" || state.nonce == req.GetResponseNonce() {
				if versionInfo != nil {
					if *versionInfo == state.version {
						// This request is an ACK.
						// Notify every observer of the ACK.
						ackObserver := s.ackObservers[typeURL]
						if ackObserver != nil {
							requestLog.Debug("notifying observers of ACK")
							ackObserver.HandleResourceVersionAck(*versionInfo, req.GetNode(), state.resourceNames, typeURL)
						} else {
							requestLog.Debug("ACK received but no observers are waiting for ACKs")
						}
					} else {
						requestLog.Warningf("NACK received for version %d; waiting for a version update before sending again", state.version)
						// Watcher will behave as if the sent version was acked.
						// Otherwise we will just be sending the same failing
						// version over and over filling logs.
						*versionInfo = state.version
					}
				}

				if state.pendingWatchCancel != nil {
					// A pending watch exists for this type URL. Cancel it to
					// start a new watch.
					requestLog.Debug("canceling pending watch")
					state.pendingWatchCancel()
				}

				respCh := make(chan *VersionedResources, 1)
				selectCases[index].Chan = reflect.ValueOf(respCh)

				ctx, cancel := context.WithCancel(ctx)
				state.pendingWatchCancel = cancel

				requestLog.Debugf("starting watch on %d resources", len(req.GetResourceNames()))
				go s.watchers[typeURL].WatchResources(ctx, typeURL, versionInfo,
					req.GetNode(), req.GetResourceNames(), respCh)
			} else {
				requestLog.Debug("received stale nonce in xDS request; ignoring request")
			}

		default: // Pending watch response.
			state := &typeStates[chosen]
			state.pendingWatchCancel()
			state.pendingWatchCancel = nil

			if !recvOK {
				streamLog.WithField(logfields.XDSTypeURL, state.typeURL).
					Error("xDS resource watch failed; terminating")
				return ErrResourceWatch
			}

			// Disabling reading from the channel after reading any from it,
			// since the watcher will close it anyway.
			selectCases[chosen].Chan = quietChValue

			resp := recv.Interface().(*VersionedResources)

			responseNonce++
			nonce := strconv.FormatUint(responseNonce, 10)

			responseLog := streamLog.WithFields(logrus.Fields{
				logfields.XDSVersionInfo: resp.Version,
				logfields.XDSCanary:      resp.Canary,
				logfields.XDSTypeURL:     state.typeURL,
				logfields.XDSNonce:       nonce,
			})

			resources := make([]*any.Any, len(resp.Resources))

			// Marshall the resources into protobuf's Any type.
			for i, res := range resp.Resources {
				data, err := proto.Marshal(res)
				if err != nil {
					responseLog.WithError(err).Errorf("error marshalling xDS response (%d resources)", len(resp.Resources))
					return err
				}
				resources[i] = &any.Any{
					TypeUrl: state.typeURL,
					Value:   data,
				}
			}

			responseLog.Infof("sending xDS response with %d resources", len(resp.Resources))

			out := &envoy_api_v2.DiscoveryResponse{
				VersionInfo: strconv.FormatUint(resp.Version, 10),
				Resources:   resources,
				Canary:      resp.Canary,
				TypeUrl:     state.typeURL,
				Nonce:       nonce,
			}
			err := stream.Send(out)
			if err != nil {
				return err
			}

			state.nonce = nonce
			state.version = resp.Version
			state.resourceNames = resp.ResourceNames
		}
	}
}
<commit_msg>xds: Change xDS response log level from Info to Debug<commit_after>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package xds

import (
	"context"
	"errors"
	"io"
	"reflect"
	"strconv"
	"sync/atomic"
	"time"

	envoy_api_v2 "github.com/cilium/cilium/pkg/envoy/envoy/api/v2"
	"github.com/cilium/cilium/pkg/logging/logfields"

	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes/any"
	"github.com/sirupsen/logrus"
)

const (
	// AnyTypeURL is the default type URL to use for ADS resource sets.
	AnyTypeURL = ""
)

var (
	// ErrNoADSTypeURL is the error returned when receiving a request without
	// a type URL from an ADS stream.
	ErrNoADSTypeURL = errors.New("type URL is required for ADS")

	// ErrUnknownTypeURL is the error returned when receiving a request with
	// an unknown type URL.
	ErrUnknownTypeURL = errors.New("unknown type URL")

	// ErrInvalidVersionInfo is the error returned when receiving a request
	// with a version info that is not a positive integer.
	ErrInvalidVersionInfo = errors.New("invalid version info")

	// ErrResourceWatch is the error returned whenever an internal error
	// occurs while waiting for new versions of resources.
	ErrResourceWatch = errors.New("resource watch failed")
)

// Server implements the handling of xDS streams.
type Server struct {
	// watchers maps each supported type URL to its corresponding resource
	// watcher.
	watchers map[string]*ResourceWatcher

	// ackObservers maps each supported type URL to its corresponding observer
	// of ACKs received from Envoy nodes.
	ackObservers map[string]ResourceVersionAckObserver

	// lastStreamID is the identifier of the last processed stream.
	// It is incremented atomically when starting the handling of a new stream.
	lastStreamID uint64
}

// ResourceTypeConfiguration is the configuration of the XDS server for a
// resource type.
type ResourceTypeConfiguration struct {
	// Source contains the resources of this type.
	Source ObservableResourceSource

	// AckObserver is called back whenever a node acknowledges having applied a
	// version of the resources of this type.
	AckObserver ResourceVersionAckObserver
}

// NewServer creates an xDS gRPC stream handler using the given resource
// sources.
// types maps each supported resource type URL to its corresponding resource
// source and ACK observer.
func NewServer(resourceTypes map[string]*ResourceTypeConfiguration,
	resourceAccessTimeout time.Duration) *Server {
	watchers := make(map[string]*ResourceWatcher, len(resourceTypes))
	ackObservers := make(map[string]ResourceVersionAckObserver, len(resourceTypes))
	for typeURL, resType := range resourceTypes {
		w := NewResourceWatcher(typeURL, resType.Source, resourceAccessTimeout)
		resType.Source.AddResourceVersionObserver(w)
		watchers[typeURL] = w

		if resType.AckObserver != nil {
			ackObservers[typeURL] = resType.AckObserver
		}
	}

	// TODO: Unregister the watchers when stopping the server.

	return &Server{watchers: watchers, ackObservers: ackObservers}
}

func getXDSRequestFields(req *envoy_api_v2.DiscoveryRequest) logrus.Fields {
	return logrus.Fields{
		logfields.XDSVersionInfo: req.GetVersionInfo(),
		logfields.XDSClientNode:  req.GetNode(),
		logfields.XDSTypeURL:     req.GetTypeUrl(),
		logfields.XDSNonce:       req.GetResponseNonce(),
	}
}

// HandleRequestStream receives and processes the requests from an xDS stream.
func (s *Server) HandleRequestStream(ctx context.Context, stream Stream, defaultTypeURL string) error {
	// increment stream count
	streamID := atomic.AddUint64(&s.lastStreamID, 1)

	streamLog := log.WithField(logfields.XDSStreamID, streamID)

	reqCh := make(chan *envoy_api_v2.DiscoveryRequest)

	stopRecv := make(chan struct{})
	defer close(stopRecv)

	go func() {
		defer close(reqCh)
		for {
			req, err := stream.Recv()
			if err != nil {
				if err == io.EOF {
					streamLog.Debug("xDS stream closed")
				} else {
					streamLog.WithError(err).Error("error while receiving request from xDS stream")
				}
				return
			}
			if req == nil {
				streamLog.Error("received nil request from xDS stream; stopping xDS stream handling")
				return
			}
			if req.GetTypeUrl() == "" {
				req.TypeUrl = defaultTypeURL
			}
			streamLog.WithFields(getXDSRequestFields(req)).Debug("received request from xDS stream")
			select {
			case <-stopRecv:
				streamLog.Debug("stopping xDS stream handling")
				return
			case reqCh <- req:
			}
		}
	}()

	return s.processRequestStream(ctx, streamLog, stream, reqCh, defaultTypeURL)
}

// perTypeStreamState is the state maintained per resource type for each
// xDS stream.
type perTypeStreamState struct {
	// typeURL identifies the resource type.
	typeURL string

	// pendingWatchCancel is a pending watch on this resource type.
	// If nil, no watch is pending.
	pendingWatchCancel context.CancelFunc

	// nonce is the nonce sent in the last response to a request for this
	// resource type.
	nonce string

	// version is the last version sent. This is needed so that we'll know
	// if a new request is an ACK (VersionInfo matches current version), or a NACK
	// (VersionInfo matches an earlier version).
	version uint64

	// resourceNames is the list of names of resources sent in the last
	// response to a request for this resource type.
	resourceNames []string
}

// processRequestStream processes the requests in an xDS stream from a channel.
func (s *Server) processRequestStream(ctx context.Context, streamLog *logrus.Entry, stream Stream,
	reqCh <-chan *envoy_api_v2.DiscoveryRequest, defaultTypeURL string) error {
	// The request state for every type URL.
	typeStates := make([]perTypeStreamState, len(s.watchers))
	defer func() {
		for _, state := range typeStates {
			if state.pendingWatchCancel != nil {
				state.pendingWatchCancel()
			}
		}
	}()

	// A map of a resource type's URL to the corresponding index in typeStates
	// for the resource type.
	typeIndexes := make(map[string]int, len(typeStates))

	// The set of channels to select from. Since the set of channels is
	// dynamic, we use reflection for selection.
	// The indexes in selectCases from 0 to len(typeStates)-1 match the indexes
	// in typeStates.
	selectCases := make([]reflect.SelectCase, len(typeStates)+2)

	// The last select case index is always the request channel.
	reqChIndex := len(selectCases) - 1
	selectCases[reqChIndex] = reflect.SelectCase{
		Dir:  reflect.SelectRecv,
		Chan: reflect.ValueOf(reqCh),
	}

	// The next-to-last select case is the context's Done channel.
	doneChIndex := reqChIndex - 1
	selectCases[doneChIndex] = reflect.SelectCase{
		Dir:  reflect.SelectRecv,
		Chan: reflect.ValueOf(ctx.Done()),
	}

	// Initially there are no pending watches, so just select a dead channel
	// that will never be selected.
	quietCh := make(chan *VersionedResources)
	defer close(quietCh)
	quietChValue := reflect.ValueOf(quietCh)

	i := 0
	for typeURL := range s.watchers {
		typeStates[i] = perTypeStreamState{
			typeURL: typeURL,
		}

		selectCases[i] = reflect.SelectCase{
			Dir:  reflect.SelectRecv,
			Chan: quietChValue,
		}

		typeIndexes[typeURL] = i

		i++
	}

	streamLog.Info("starting xDS stream processing")

	// The last nonce returned in a response in this stream.
	var responseNonce uint64

	for {
		// Process either a new request from the xDS stream or a response
		// from the resource watcher.
		chosen, recv, recvOK := reflect.Select(selectCases)

		switch chosen {
		case doneChIndex: // Context got canceled.
			streamLog.WithError(ctx.Err()).Error("xDS stream context canceled")
			return ctx.Err()

		case reqChIndex: // Request received from the stream.
			if !recvOK {
				streamLog.Info("xDS stream closed")
				return nil
			}

			req := recv.Interface().(*envoy_api_v2.DiscoveryRequest)

			requestLog := streamLog.WithFields(getXDSRequestFields(req))

			// Ensure that the version info is a string that was sent by this
			// server or the empty string (the first request in a stream should
			// always have an empty version info).
			var versionInfo *uint64
			if req.GetVersionInfo() != "" {
				versionInfoVal, err := strconv.ParseUint(req.VersionInfo, 10, 64)
				if err != nil {
					requestLog.Errorf("invalid version info in xDS request, not a uint64")
					return ErrInvalidVersionInfo
				}
				versionInfo = &versionInfoVal
			}

			typeURL := req.GetTypeUrl()
			if defaultTypeURL == AnyTypeURL && typeURL == "" {
				requestLog.Error("no type URL given in ADS request")
				return ErrNoADSTypeURL
			}

			index, exists := typeIndexes[typeURL]

			if !exists {
				requestLog.Error("unknown type URL in xDS request")
				return ErrUnknownTypeURL
			}

			state := &typeStates[index]

			// Only handle a request if the nonce is valid, i.e. if it's
			// the first request (which nonce is "") or if it's the nonce of
			// the last response that was sent.
			//
			// If the nonce is different (stale), the client hasn't processed
			// the last response yet. Ignore every request until it processes
			// that response and sends a request with that response's nonce.
			if state.nonce == "" || state.nonce == req.GetResponseNonce() {
				if versionInfo != nil {
					if *versionInfo == state.version {
						// This request is an ACK.
						// Notify every observer of the ACK.
						ackObserver := s.ackObservers[typeURL]
						if ackObserver != nil {
							requestLog.Debug("notifying observers of ACK")
							ackObserver.HandleResourceVersionAck(*versionInfo, req.GetNode(), state.resourceNames, typeURL)
						} else {
							requestLog.Debug("ACK received but no observers are waiting for ACKs")
						}
					} else {
						requestLog.Warningf("NACK received for version %d; waiting for a version update before sending again", state.version)
						// Watcher will behave as if the sent version was acked.
						// Otherwise we will just be sending the same failing
						// version over and over filling logs.
						*versionInfo = state.version
					}
				}

				if state.pendingWatchCancel != nil {
					// A pending watch exists for this type URL. Cancel it to
					// start a new watch.
					requestLog.Debug("canceling pending watch")
					state.pendingWatchCancel()
				}

				respCh := make(chan *VersionedResources, 1)
				selectCases[index].Chan = reflect.ValueOf(respCh)

				ctx, cancel := context.WithCancel(ctx)
				state.pendingWatchCancel = cancel

				requestLog.Debugf("starting watch on %d resources", len(req.GetResourceNames()))
				go s.watchers[typeURL].WatchResources(ctx, typeURL, versionInfo,
					req.GetNode(), req.GetResourceNames(), respCh)
			} else {
				requestLog.Debug("received stale nonce in xDS request; ignoring request")
			}

		default: // Pending watch response.
			state := &typeStates[chosen]
			state.pendingWatchCancel()
			state.pendingWatchCancel = nil

			if !recvOK {
				streamLog.WithField(logfields.XDSTypeURL, state.typeURL).
					Error("xDS resource watch failed; terminating")
				return ErrResourceWatch
			}

			// Disabling reading from the channel after reading any from it,
			// since the watcher will close it anyway.
			selectCases[chosen].Chan = quietChValue

			resp := recv.Interface().(*VersionedResources)

			responseNonce++
			nonce := strconv.FormatUint(responseNonce, 10)

			responseLog := streamLog.WithFields(logrus.Fields{
				logfields.XDSVersionInfo: resp.Version,
				logfields.XDSCanary:      resp.Canary,
				logfields.XDSTypeURL:     state.typeURL,
				logfields.XDSNonce:       nonce,
			})

			resources := make([]*any.Any, len(resp.Resources))

			// Marshall the resources into protobuf's Any type.
			for i, res := range resp.Resources {
				data, err := proto.Marshal(res)
				if err != nil {
					responseLog.WithError(err).Errorf("error marshalling xDS response (%d resources)", len(resp.Resources))
					return err
				}
				resources[i] = &any.Any{
					TypeUrl: state.typeURL,
					Value:   data,
				}
			}

			responseLog.Debugf("sending xDS response with %d resources", len(resp.Resources))

			out := &envoy_api_v2.DiscoveryResponse{
				VersionInfo: strconv.FormatUint(resp.Version, 10),
				Resources:   resources,
				Canary:      resp.Canary,
				TypeUrl:     state.typeURL,
				Nonce:       nonce,
			}
			err := stream.Send(out)
			if err != nil {
				return err
			}

			state.nonce = nonce
			state.version = resp.Version
			state.resourceNames = resp.ResourceNames
		}
	}
}
<|endoftext|>


<commit_before>package etcdutil

import (
	"log"
	"math/rand"
	"path"
	"strconv"
	"time"

	"github.com/coreos/go-etcd/etcd"
)

// heartbeat to etcd cluster until stop
func Heartbeat(client *etcd.Client, name string, taskID uint64, interval time.Duration, stop chan struct{}) error {
	for {
		_, err := client.Set(TaskHealthyPath(name, taskID), "health", computeTTL(interval))
		if err != nil {
			return err
		}
		select {
		case <-time.After(interval):
		case <-stop:
			return nil
		}
	}
}

// detect failure of the given taskID
func DetectFailure(client *etcd.Client, name string, stop chan bool) error {
	receiver := make(chan *etcd.Response, 1)
	go client.Watch(HealthyPath(name), 0, true, receiver, stop)
	for resp := range receiver {
		if resp.Action != "expire" && resp.Action != "delete" {
			continue
		}
		if err := ReportFailure(client, name, path.Base(resp.Node.Key)); err != nil {
			return err
		}
	}
	return nil
}

// report failure to etcd cluster
// If a framework detects a failure, it tries to report failure to /FreeTasks/{taskID}
func ReportFailure(client *etcd.Client, name, failedTask string) error {
	_, err := client.Set(FreeTaskPath(name, failedTask), "failed", 0)
	return err
}

// WaitFreeTask blocks until it gets a hint of free task
func WaitFreeTask(client *etcd.Client, name string, logger *log.Logger) (uint64, error) {
	slots, err := client.Get(FreeTaskDir(name), false, true)
	if err != nil {
		return 0, err
	}
	if total := len(slots.Node.Nodes); total > 0 {
		ri := rand.Intn(total)
		s := slots.Node.Nodes[ri]
		idStr := path.Base(s.Key)
		id, err := strconv.ParseUint(idStr, 0, 64)
		if err != nil {
			return 0, err
		}
		logger.Printf("got free task %v, randomly choose %d to try...", ListKeys(slots.Node.Nodes), ri)
		return id, nil
	}

	watchIndex := slots.EtcdIndex + 1
	respChan := make(chan *etcd.Response, 1)
	go func() {
		for {
			logger.Printf("start to wait failure at index %d", watchIndex)
			resp, err := client.Watch(FreeTaskDir(name), watchIndex, true, nil, nil)
			if err != nil {
				logger.Printf("WARN: WaitFailure watch failed: %v", err)
				return
			}
			if resp.Action == "set" {
				respChan <- resp
				return
			}
			watchIndex = resp.EtcdIndex + 1
		}
	}()
	var resp *etcd.Response
	var waitTime uint64 = 0
	for {
		select {
		case resp = <-respChan:
			idStr := path.Base(resp.Node.Key)
			id, err := strconv.ParseUint(idStr, 10, 64)
			if err != nil {
				return 0, err
			}
			return id, nil
		case <-time.After(10 * time.Second):
			waitTime++
			logger.Printf("Node already wait failure for %d0s", waitTime)
		}
	}

}

func computeTTL(interval time.Duration) uint64 {
	if interval/time.Second < 1 {
		return 3
	}
	return 3 * uint64(interval/time.Second)
}
<commit_msg>improve: task failure would only lead to health node expiration, we don't need detect 'delete' event<commit_after>package etcdutil

import (
	"log"
	"math/rand"
	"path"
	"strconv"
	"time"

	"github.com/coreos/go-etcd/etcd"
)

// heartbeat to etcd cluster until stop
func Heartbeat(client *etcd.Client, name string, taskID uint64, interval time.Duration, stop chan struct{}) error {
	for {
		_, err := client.Set(TaskHealthyPath(name, taskID), "health", computeTTL(interval))
		if err != nil {
			return err
		}
		select {
		case <-time.After(interval):
		case <-stop:
			return nil
		}
	}
}

// detect failure of the given taskID
func DetectFailure(client *etcd.Client, name string, stop chan bool) error {
	receiver := make(chan *etcd.Response, 1)
	go client.Watch(HealthyPath(name), 0, true, receiver, stop)
	for resp := range receiver {
		if resp.Action != "expire" {
			continue
		}
		if err := ReportFailure(client, name, path.Base(resp.Node.Key)); err != nil {
			return err
		}
	}
	return nil
}

// report failure to etcd cluster
// If a framework detects a failure, it tries to report failure to /FreeTasks/{taskID}
func ReportFailure(client *etcd.Client, name, failedTask string) error {
	_, err := client.Set(FreeTaskPath(name, failedTask), "failed", 0)
	return err
}

// WaitFreeTask blocks until it gets a hint of free task
func WaitFreeTask(client *etcd.Client, name string, logger *log.Logger) (uint64, error) {
	slots, err := client.Get(FreeTaskDir(name), false, true)
	if err != nil {
		return 0, err
	}
	if total := len(slots.Node.Nodes); total > 0 {
		ri := rand.Intn(total)
		s := slots.Node.Nodes[ri]
		idStr := path.Base(s.Key)
		id, err := strconv.ParseUint(idStr, 0, 64)
		if err != nil {
			return 0, err
		}
		logger.Printf("got free task %v, randomly choose %d to try...", ListKeys(slots.Node.Nodes), ri)
		return id, nil
	}

	watchIndex := slots.EtcdIndex + 1
	respChan := make(chan *etcd.Response, 1)
	go func() {
		for {
			logger.Printf("start to wait failure at index %d", watchIndex)
			resp, err := client.Watch(FreeTaskDir(name), watchIndex, true, nil, nil)
			if err != nil {
				logger.Printf("WARN: WaitFailure watch failed: %v", err)
				return
			}
			if resp.Action == "set" {
				respChan <- resp
				return
			}
			watchIndex = resp.EtcdIndex + 1
		}
	}()
	var resp *etcd.Response
	var waitTime uint64 = 0
	for {
		select {
		case resp = <-respChan:
			idStr := path.Base(resp.Node.Key)
			id, err := strconv.ParseUint(idStr, 10, 64)
			if err != nil {
				return 0, err
			}
			return id, nil
		case <-time.After(10 * time.Second):
			waitTime++
			logger.Printf("Node already wait failure for %d0s", waitTime)
		}
	}

}

func computeTTL(interval time.Duration) uint64 {
	if interval/time.Second < 1 {
		return 3
	}
	return 3 * uint64(interval/time.Second)
}
<|endoftext|>


<commit_before>package modules

import (
	"math/rand"
	"strings"

	"github.com/pajlada/pajbot2/pkg"
)

type Giveaway struct {
	server *server

	state string

	entrants []string

	Sender pkg.Sender
}

func NewGiveaway(sender pkg.Sender) *Giveaway {
	return &Giveaway{
		server: &_server,
		state:  "inactive",
		Sender: sender,
	}
}

func (m Giveaway) Name() string {
	return "Giveaway"
}

func (m Giveaway) Register() error {
	return nil
}

const forsen25ID = "1072584"
const pajlada25ID = "908917"
const pajaWID = "80481"

func (m Giveaway) OnWhisper(bot pkg.Sender, user pkg.User, message pkg.Message) error {
	return nil
}

func (m *Giveaway) OnMessage(bot pkg.Sender, channel pkg.Channel, user pkg.User, message pkg.Message, action pkg.Action) error {
	// if channel.GetChannel() != "forsen" {
	// 	return nil
	// }

	const giveawayEmote = forsen25ID

	text := message.GetText()

	// Commands
	if user.IsModerator() || user.GetName() == "forsen" || user.GetName() == "pajlada" {
		if strings.HasPrefix(text, "!25start") {
			if m.state == "inactive" {
				m.state = "started"
				m.entrants = []string{}
				m.Sender.Say(channel, "Started giveaway")

				return nil
			} else {
				m.Sender.Say(channel, "Giveaway already started")
				return nil
			}
		}

		if strings.HasPrefix(text, "!25stop") {
			if m.state == "started" {
				m.state = "inactive"
				m.Sender.Say(channel, "Stopped accepting people into the giveaway")

				return nil
			}
		}

		if strings.HasPrefix(text, "!25draw") {
			if len(m.entrants) == 0 {
				m.Sender.Say(channel, "No one has joined the giveaway")
				return nil
			}

			winnerIndex := rand.Intn(len(m.entrants))
			winnerUsername := m.entrants[winnerIndex]
			m.Sender.Say(channel, winnerUsername+" just won the sub emote giveaway PogChamp")

			m.entrants = append(m.entrants[:winnerIndex], m.entrants[winnerIndex+1:]...)

			return nil
		}
	}

	if m.state == "started" {
		enterGiveaway := false

		reader := message.GetTwitchReader()
		for reader.Next() {
			emote := reader.Get()
			if emote.GetID() == giveawayEmote {
				// m.Sender.Say(channel, fmt.Sprintf("%#v", emote))
				// User wants to enter the giveaway
				enterGiveaway = true
				break
			}
		}

		if enterGiveaway {
			for _, entrant := range m.entrants {
				if entrant == user.GetName() {
					// User has already joined
					return nil
				}

			}
			m.entrants = append(m.entrants, user.GetName())

			m.Sender.Say(channel, "@"+user.GetName()+", you have been entered into the sub emote giveaway")
			return nil
		}
	}

	return nil
}
<commit_msg>update forsens 25 emote id KKona<commit_after>package modules

import (
	"math/rand"
	"strings"

	"github.com/pajlada/pajbot2/pkg"
)

type Giveaway struct {
	server *server

	state string

	entrants []string

	Sender pkg.Sender
}

func NewGiveaway(sender pkg.Sender) *Giveaway {
	return &Giveaway{
		server: &_server,
		state:  "inactive",
		Sender: sender,
	}
}

func (m Giveaway) Name() string {
	return "Giveaway"
}

func (m Giveaway) Register() error {
	return nil
}

const forsen25ID = "1171485"
const pajlada25ID = "908917"
const pajaWID = "80481"

func (m Giveaway) OnWhisper(bot pkg.Sender, user pkg.User, message pkg.Message) error {
	return nil
}

func (m *Giveaway) OnMessage(bot pkg.Sender, channel pkg.Channel, user pkg.User, message pkg.Message, action pkg.Action) error {
	// if channel.GetChannel() != "forsen" {
	// 	return nil
	// }

	const giveawayEmote = forsen25ID

	text := message.GetText()

	// Commands
	if user.IsModerator() || user.GetName() == "forsen" || user.GetName() == "pajlada" {
		if strings.HasPrefix(text, "!25start") {
			if m.state == "inactive" {
				m.state = "started"
				m.entrants = []string{}
				m.Sender.Say(channel, "Started giveaway")

				return nil
			} else {
				m.Sender.Say(channel, "Giveaway already started")
				return nil
			}
		}

		if strings.HasPrefix(text, "!25stop") {
			if m.state == "started" {
				m.state = "inactive"
				m.Sender.Say(channel, "Stopped accepting people into the giveaway")

				return nil
			}
		}

		if strings.HasPrefix(text, "!25draw") {
			if len(m.entrants) == 0 {
				m.Sender.Say(channel, "No one has joined the giveaway")
				return nil
			}

			winnerIndex := rand.Intn(len(m.entrants))
			winnerUsername := m.entrants[winnerIndex]
			m.Sender.Say(channel, winnerUsername+" just won the sub emote giveaway PogChamp")

			m.entrants = append(m.entrants[:winnerIndex], m.entrants[winnerIndex+1:]...)

			return nil
		}
	}

	if m.state == "started" {
		enterGiveaway := false

		reader := message.GetTwitchReader()
		for reader.Next() {
			emote := reader.Get()
			if emote.GetID() == giveawayEmote {
				// m.Sender.Say(channel, fmt.Sprintf("%#v", emote))
				// User wants to enter the giveaway
				enterGiveaway = true
				break
			}
		}

		if enterGiveaway {
			for _, entrant := range m.entrants {
				if entrant == user.GetName() {
					// User has already joined
					return nil
				}

			}
			m.entrants = append(m.entrants, user.GetName())

			m.Sender.Say(channel, "@"+user.GetName()+", you have been entered into the sub emote giveaway")
			return nil
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package getprofile

import (
    "regexp"
    "os/exec"
    "path"
    "path/filepath"
    "os"
    "errors"
)

type gitSyncer struct{}

var gitSyncerInstance *gitSyncer = &gitSyncer{}

func (sync *gitSyncer) Supports(repoUrl string) bool {
    if matched, err := regexp.MatchString(".+@.+:.+", repoUrl); err != nil {
        dbgf("Error checking gitSyncer.Supports(%v)", repoUrl)
        return false
    } else {
        return matched
    }
}

func (sync *gitSyncer) Init() error {
    if repoUrl, err := getConfig(); err != nil {
        return err
    } else if _, err := os.Stat(path.Join(repoPath, ".git")); os.IsNotExist(err) {
        return exec.Command("git", "clone", repoUrl, path.Join(basePath, "repo")).Run()
    } else {
        return nil // Already cloned
    }
}

func (sync *gitSyncer) Track(absPath, relPath string) error {
    copyToRepo(absPath, relPath)

    cmd := exec.Command("git", "add", relPath)
    cmd.Dir = repoPath
    return cmd.Run()
}

func (sync *gitSyncer) Untrack(relPath string) error {
    cmd := exec.Command("git", "rm", "-f", relPath)
    cmd.Dir = repoPath
    return cmd.Run()
}

func (sync *gitSyncer) Out() error {
    inf("Sending updates out")
    if err := filepath.Walk(repoPath, makeWalkFunc(copyToRepo)); err != nil {
        return err
    } else {
        repoPath := path.Join(basePath, "repo")

        dbg("Command:", "git", "diff", "HEAD", "--quiet")
        cmd := exec.Command("git", "diff", "HEAD", "--quiet")
        cmd.Dir = repoPath
        if err := cmd.Run(); err != nil {
            if execExitStatus(err, 128) {
                dbg("New git repo. Will try to continue.")
            } else {
                return err
            }
        }

        if cmd.ProcessState.Success() {
            return nil // No changes
        } else if _, err := gitRepoExec("git", "add", "-u"); err != nil {
            return err
        } else if _, err := gitRepoExec("git", "commit", "-m", "auto-commit"); err != nil {
            return err
        } else {
            _, err = gitRepoExec("git", "push")
            return err
        }
    }
}

func (sync *gitSyncer) In() error {
    inf("Copying updates in")
    if prevSha, err := gitSha(); err != nil {
        return err
    } else if _, err := gitRepoExec("git", "pull"); err != nil {
        return err
    } else if nowSha, err := gitSha(); err != nil {
        return err
    } else if prevSha == nowSha {
        return nil
    } else {
        return filepath.Walk(path.Join(basePath, "repo"), makeWalkFunc(copyToLocal))
    }
}

func makeWalkFunc(processor func(absPath, relPath string) error) filepath.WalkFunc {
    return func(absPath string, info os.FileInfo, err error) error {
        if err != nil {
            return err
        } else if info.IsDir() {
            return nil
        } else if relPath, err := filepath.Rel(repoPath, absPath); err != nil {
            return err
        } else if !filepath.HasPrefix(relPath, ".git/") {
            return processor(absPath, relPath)
        } else {
            return nil
        }
    }
}

func gitRepoExec(cmdToks ...string) (string, error) {
    dbgf("Command: %v", cmdToks)
    cmd := exec.Command(cmdToks[0], cmdToks[1:]...)
    cmd.Dir = repoPath
    if bytes, err := cmd.Output(); err != nil {
        return string(bytes), err
    } else {
        return string(bytes), nil
    }
}

func gitSha() (string, error) {
    return gitRepoExec("git", "rev-parse", "HEAD")
}

func copyToRepo(absPath, relPath string) error {
    inf("Copy to repo:", relPath)
    src := path.Join(homePath, relPath)
    dest := path.Join(repoPath, relPath)
    if err := os.MkdirAll(path.Dir(dest), 0700); err != nil {
        return err
    }

    dbg("Command: cp", src, dest)
    return exec.Command("cp", src, dest).Run()
}

func copyToLocal(absPath, relPath string) error {
    inf("Copy to local:", relPath)
    return errors.New("TODO") // TODO
}

<commit_msg>only copy files that exist to repo<commit_after>package getprofile

import (
    "regexp"
    "os/exec"
    "path"
    "path/filepath"
    "os"
    "errors"
    "fmt"
)

type gitSyncer struct{}

var gitSyncerInstance *gitSyncer = &gitSyncer{}

func (sync *gitSyncer) Supports(repoUrl string) bool {
    if matched, err := regexp.MatchString(".+@.+:.+", repoUrl); err != nil {
        dbgf("Error checking gitSyncer.Supports(%v)", repoUrl)
        return false
    } else {
        return matched
    }
}

func (sync *gitSyncer) Init() error {
    if repoUrl, err := getConfig(); err != nil {
        return err
    } else if _, err := os.Stat(path.Join(repoPath, ".git")); os.IsNotExist(err) {
        return exec.Command("git", "clone", repoUrl, path.Join(basePath, "repo")).Run()
    } else {
        return nil // Already cloned
    }
}

func (sync *gitSyncer) Track(absPath, relPath string) error {
    copyToRepo(absPath, relPath)

    cmd := exec.Command("git", "add", relPath)
    cmd.Dir = repoPath
    return cmd.Run()
}

func (sync *gitSyncer) Untrack(relPath string) error {
    cmd := exec.Command("git", "rm", "-f", relPath)
    cmd.Dir = repoPath
    return cmd.Run()
}

func (sync *gitSyncer) Out() error {
    inf("Sending updates out")
    if err := filepath.Walk(repoPath, makeWalkFunc(copyToRepo)); err != nil {
        return err
    } else {
        repoPath := path.Join(basePath, "repo")

        dbg("Command:", "git", "diff", "HEAD", "--quiet")
        cmd := exec.Command("git", "diff", "HEAD", "--quiet")
        cmd.Dir = repoPath
        if err := cmd.Run(); err != nil {
            if execExitStatus(err, 128) {
                dbg("New git repo. Will try to continue.")
            } else {
                return err
            }
        }

        if cmd.ProcessState.Success() {
            return nil // No changes
        } else if _, err := gitRepoExec("git", "add", "-u"); err != nil {
            return err
        } else if _, err := gitRepoExec("git", "commit", "-m", "auto-commit"); err != nil {
            return err
        } else {
            _, err = gitRepoExec("git", "push")
            return err
        }
    }
}

func (sync *gitSyncer) In() error {
    inf("Copying updates in")
    if prevSha, err := gitSha(); err != nil {
        return err
    } else if _, err := gitRepoExec("git", "pull"); err != nil {
        return err
    } else if nowSha, err := gitSha(); err != nil {
        return err
    } else if prevSha == nowSha {
        return nil
    } else {
        return filepath.Walk(path.Join(basePath, "repo"), makeWalkFunc(copyToLocal))
    }
}

func makeWalkFunc(processor func(absPath, relPath string) error) filepath.WalkFunc {
    return func(absPath string, info os.FileInfo, err error) error {
        if err != nil {
            return err
        } else if info.IsDir() {
            return nil
        } else if relPath, err := filepath.Rel(repoPath, absPath); err != nil {
            return err
        } else if !filepath.HasPrefix(relPath, ".git/") {
            return processor(absPath, relPath)
        } else {
            return nil
        }
    }
}

func gitRepoExec(cmdToks ...string) (string, error) {
    dbgf("Command: %v", cmdToks)
    cmd := exec.Command(cmdToks[0], cmdToks[1:]...)
    cmd.Dir = repoPath
    if bytes, err := cmd.Output(); err != nil {
        return string(bytes), err
    } else {
        return string(bytes), nil
    }
}

func gitSha() (string, error) {
    return gitRepoExec("git", "rev-parse", "HEAD")
}

func copyToRepo(absPath, relPath string) error {
    inf("Copy to repo:", relPath)
    src := path.Join(homePath, relPath)
    dest := path.Join(repoPath, relPath)
    if err := os.MkdirAll(path.Dir(dest), 0700); err != nil {
        return err
    }

    cmd := fmt.Sprintf("[ -e '%v' ] && cp '%v' '%v'", src, src, dest)
    dbg("Command:", cmd)
    return exec.Command("bash", "-c", cmd).Run()
}

func copyToLocal(absPath, relPath string) error {
    inf("Copy to local:", relPath)
    return errors.New("TODO") // TODO
}

<|endoftext|>