profile
viewpoint
Tom Wieczorek twz123 @yieldlab Germany

dhiller/readmethenewz 0

Narrates the latest news items from selected news feeds

twz123/Amazon-S3-FileSystem-NIO2 0

An S3 File System Provider for Java 7

twz123/argo 0

Argo Workflows: Get stuff done with Kubernetes.

twz123/argo-cd 0

Declarative Continuous Delivery for Kubernetes

twz123/argocd-playground 0

Toying around with ArgoCD

twz123/async-http-client 0

Asynchronous Http and WebSocket Client library for Java

twz123/awesome-kubernetes 0

A curated list for awesome kubernetes sources :ship::tada:

twz123/Bitbucket-EagerPR-Updates 0

Eagarly update bitbucket internal pull-request references upon PR events

twz123/broken-assembly 0

Maven Assembly Plugin may create broken JAR files.

twz123/click-docker 0

Builds the https://github.com/databricks/click binary and stores it into a Docker container

Pull request review commentcockroachdb/cockroach-operator

Implementing resizing PVCs

+/*+Copyright 2020 The Cockroach Authors++Licensed under the Apache License, Version 2.0 (the "License");+you may not use this file except in compliance with the License.+You may obtain a copy of the License at++    https://www.apache.org/licenses/LICENSE-2.0++Unless required by applicable law or agreed to in writing, software+distributed under the License is distributed on an "AS IS" BASIS,+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+See the License for the specific language governing permissions and+limitations under the License.+*/++package actor++import (+	"context"+	"fmt"+	"time"++	api "github.com/cockroachdb/cockroach-operator/api/v1alpha1"+	"github.com/cockroachdb/cockroach-operator/pkg/condition"+	"github.com/cockroachdb/cockroach-operator/pkg/resource"+	"github.com/cockroachdb/cockroach-operator/pkg/update"+	"github.com/cockroachdb/errors"+	appsv1 "k8s.io/api/apps/v1"+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"+	"k8s.io/apimachinery/pkg/runtime"+	kubetypes "k8s.io/apimachinery/pkg/types"+	"k8s.io/client-go/kubernetes"+	"k8s.io/client-go/rest"+	"sigs.k8s.io/controller-runtime/pkg/client"+)++// newResizePVC creates and returns a new resizePVC struct+func newResizePVC(scheme *runtime.Scheme, cl client.Client, config *rest.Config) Actor {+	return &resizePVC{+		action: newAction("resize_pvc", scheme, cl),+		config: config,+	}+}++// resizePVC resizes a PVC+type resizePVC struct {+	action++	config *rest.Config+}++// Handles returns true if the DB is initialized+func (rp *resizePVC) Handles(conds []api.ClusterCondition) bool {+	return condition.False(api.NotInitializedCondition, conds)+}++// Act in this implementation resizes PVC volumes of a CR sts.+func (rp *resizePVC) Act(ctx context.Context, cluster *resource.Cluster) error {+	log := rp.log.WithValues("CrdbCluster", cluster.ObjectKey())++	// If we do not have a volume claim we do not have PVCs+	if cluster.Spec().DataStore.VolumeClaim == nil {+		log.Info("Skipping PVC resize as VolumeClaim does not exist")+		return nil+	}++	// Get the sts and compare the sts size to the size in the CR+	key := kubetypes.NamespacedName{+		Namespace: cluster.Namespace(),+		Name:      cluster.StatefulSetName(),+	}+	statefulSet := &appsv1.StatefulSet{}+	if err := rp.client.Get(ctx, key, statefulSet); err != nil {+		return errors.Wrap(err, "failed to fetch statefulset")+	}++	if statefulSetIsUpdating(statefulSet) {+		return NotReadyErr{Err: errors.New("statefulset is updating, waiting for the update to finish")}+	}++	// Maybe this should be an error since we should not have this, but I wanted to check anyways+	if len(statefulSet.Spec.VolumeClaimTemplates) == 0 {+		log.Info("Skipping PVC resize as PVCs do not exist")+		return nil+	}++	clientset, err := kubernetes.NewForConfig(rp.config)+	if err != nil {+		return errors.Wrapf(err, "failed to create kubernetes clientset")+	}++	stsStorageSizeDeployed := statefulSet.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage()+	stsStorageSizeSet := cluster.Spec().DataStore.VolumeClaim.PersistentVolumeClaimSpec.Resources.Requests.Storage()++	// If the sizes match do not resize+	if stsStorageSizeDeployed.Equal(stsStorageSizeSet.DeepCopy()) {+		log.Info("Skipping PVC resize as sizes match")+		return nil+	}++	log.Info("Starting PVC resize")++	// Find all of the PVCs and resize them+	if err := rp.findAndResizePVC(ctx, statefulSet, cluster, clientset); err != nil {+		return errors.Wrapf(err, "updating PVCs for statefulset %s.%s", cluster.Namespace(), cluster.StatefulSetName())+	}++	log.Info("Starting updating sts")++	// Update the STS with the correct volume size, in case more pods are created+	if err := rp.updateSts(ctx, statefulSet, cluster); err != nil {+		return errors.Wrapf(err, "updating statefulset %s.%s", cluster.Namespace(), cluster.StatefulSetName())+	}++	log.Info("Starting rolling sts")++	// Roll the entire STS in order for the Pods to resize+	if err := rp.rollSts(ctx, cluster, clientset); err != nil {+		return errors.Wrapf(err, "error restarting statefulset %s.%s", cluster.Namespace(), cluster.StatefulSetName())+	}++	log.Info("PVC resize completed")+	CancelLoop(ctx)++	return nil+}++// updateSts updates the size of an STS' VolumeClaimTemplate to match the new size in the CR.+func (rp *resizePVC) updateSts(ctx context.Context, sts *appsv1.StatefulSet, cluster *resource.Cluster) error {++	metaMutator := func(name string) metav1.ObjectMeta {+		return metav1.ObjectMeta{+			Name: resource.DataDirName,+		}++	}+	if err := cluster.Spec().DataStore.Apply(resource.DataDirName,+		resource.DbContainerName, resource.DataDirMountPath, &sts.Spec, metaMutator); err != nil {+		return err+	}++	if err := rp.client.Update(ctx, sts, &client.UpdateOptions{}); err != nil {+		return err+	}++	return nil+}++// findAndResizePVC finds all active PVCs and resizes them to the new size contained in the cluster+// definition.+func (rp *resizePVC) findAndResizePVC(ctx context.Context, sts *appsv1.StatefulSet, cluster *resource.Cluster,+	clientset *kubernetes.Clientset) error {+	// K8s doesn't provide a way to tell if a PVC or PV is currently in use by+	// a pod. However, it is safe to assume that any PVCs with an ordinal great+	// than or equal to the sts' Replicas is not in use. As only pods with with+	// an ordinal < Replicas will exist. Any PVCs with an ordinal less than+	// Replicas is in use. To detect this, we build a map of PVCs that we+	// consider to be in use and skip and PVCs that it contains+	// the name of.+	log := rp.log.WithValues("CrdbCluster", cluster.ObjectKey())+	log.Info("starting finding and resizing all PVCs")+	prefixes := make([]string, len(sts.Spec.VolumeClaimTemplates))+	pvcsToKeep := make(map[string]bool, int(*sts.Spec.Replicas)*len(sts.Spec.VolumeClaimTemplates))+	for j, pvct := range sts.Spec.VolumeClaimTemplates {+		prefixes[j] = fmt.Sprintf("%s-%s-", pvct.Name, sts.Name)++		for i := int32(0); i < *sts.Spec.Replicas; i++ {+			name := fmt.Sprintf("%s-%s-%d", pvct.Name, sts.Name, i)+			pvcsToKeep[name] = true+		}+	}++	selector, err := metav1.LabelSelectorAsSelector(sts.Spec.Selector)+	if err != nil {+		return errors.Wrap(err, "converting statefulset selector to metav1 selector")+	}++	pvcs, err := clientset.CoreV1().PersistentVolumeClaims(cluster.Namespace()).List(ctx, metav1.ListOptions{+		LabelSelector: selector.String(),+	})++	if err != nil {+		return errors.Wrap(err, "finding PVCs to for resizing")+	}++	for _, pvc := range pvcs.Items {+		// Resize PVCs that are still in use+		if pvcsToKeep[pvc.Name] {+			pvc.Spec.Resources.Requests.Storage().Set(cluster.Spec().DataStore.VolumeClaim.PersistentVolumeClaimSpec.Resources.Requests.Storage().MilliValue())+			if _, err := clientset.CoreV1().PersistentVolumeClaims(cluster.Namespace()).Update(ctx, &pvc, metav1.UpdateOptions{}); err != nil {+				return errors.Wrap(err, "error resizing PVCs")+			}+		}+	}++	log.Info("found and resized all PVCs")++	return nil+}++// rollSts performs a rolling update on the cluster.+func (rp *resizePVC) rollSts(ctx context.Context, cluster *resource.Cluster, clientset *kubernetes.Clientset) error {++	updateRoach := &update.UpdateRoach{+		StsName:      cluster.StatefulSetName(),+		StsNamespace: cluster.Namespace(),+	}++	podUpdateTimeout := 10 * time.Minute+	podMaxPollingInterval := 30 * time.Minute+	sleeper := update.NewSleeper(1 * time.Minute)++	k8sCluster := &update.UpdateCluster{+		Clientset:             clientset,+		PodUpdateTimeout:      podUpdateTimeout,+		PodMaxPollingInterval: podMaxPollingInterval,+		Sleeper:               sleeper,+	}++	return update.RollingRestart(ctx, updateRoach, k8sCluster, rp.log)

@zmalik so I tested this by hand and noticed a couple of things. We cannot resize the statefulset directly. I am still getting the status that the pods have to be restarted, but that goes away. Kinda weird. The volume does resize.

Now, this is only true for some storage classes, so we do need the code for storage classes, that won't resize, and an API value to enable resizing without Pod restarts as well.

@chrisseto do you want another API value, or should the user restart the Pods?

  1. Have API value that allows pods to skip restart on resize
  2. Add code to not have statefulset volume template update
  3. Modify checks in code that only tests the PVC size vs the cluster CR, because we cannot trust the statefulset
  4. Figure out if need to modify the volumeclaimtemplate.
kubectl edit pvc <name> # for each PVC in the StatefulSet, to increase its capacity.
kubectl delete sts --cascade=false # <name> to delete the StatefulSet and leave its pods.
kubectl apply -f <name> # to recreate the StatefulSet.
kubectl rollout restart sts <name> # to restart the pods, one at a time. During restart, the pod's PVC will be resized.
chrislovecnm

comment created time in 8 hours

Pull request review commentcockroachdb/cockroach-operator

Fixing stamping in container

 STABLE_IMAGE_REPOSITORY ${DOCKER_IMAGE_REPOSITORY:-cockroach-operator} IMAGE_REGISTRY ${DEV_REGISTRY:-us.gcr.io/chris-love-operator-playground}  CLUSTER ${K8S_CLUSTER:-gke_chris-love-operator-playground_us-central1-a_test}+GIT_REV_LIST $(git rev-list $(git rev-parse --abbrev-ref HEAD) | wc -l)

How about we name this to describe its contents. Like one of:

NUMBER_COMMITS_ON_BRANCH $(git rev-list $(git rev-parse --abbrev-ref HEAD) | wc -l)
GIT_REV_LIST_COUNT $(git rev-list $(git rev-parse --abbrev-ref HEAD) | wc -l)
chrislovecnm

comment created time in 8 hours

Pull request review commentcockroachdb/cockroach-operator

Fixing stamping in container

 container_image(     labels = {         "name": "CockroachDB Operator",         "vendor": "Cockroach Labs",-        # FIXME - labels do not support a stamped variable such as {STABLE_DOCKER_TAG}-        "version": "v1.0.0-rc.0",-        "release": "0",-        "summary": "CockroachDB is a Distributed SQL database",-        "description": "CockroachDB is a PostgreSQL wire-compatible DistributedSQL database",+        "version": "{STABLE_DOCKER_TAG}",+        "release": "{STABLE_DOCKER_TAG}",

It is another rev-list command that is supported in the older versions of git. I figured it out. PTAL

chrislovecnm

comment created time in 14 hours

issue commentcockroachdb/cockroach-operator

Test decommission feature and partial updates

yes :) with database load and more complex scenarios if it is possible

alinadonisa

comment created time in 14 hours

Pull request review commentcockroachdb/cockroach-operator

Fixing stamping in container

 container_image(     labels = {         "name": "CockroachDB Operator",         "vendor": "Cockroach Labs",-        # FIXME - labels do not support a stamped variable such as {STABLE_DOCKER_TAG}-        "version": "v1.0.0-rc.0",-        "release": "0",-        "summary": "CockroachDB is a Distributed SQL database",-        "description": "CockroachDB is a PostgreSQL wire-compatible DistributedSQL database",+        "version": "{STABLE_DOCKER_TAG}",+        "release": "{STABLE_DOCKER_TAG}",

Ah. It only works with git 2.22 and up (so post). This will work for older releases: git rev-parse --abbrev-ref HEAD

chrislovecnm

comment created time in 14 hours

Pull request review commentcockroachdb/cockroach-operator

Fixing stamping in container

 container_image(     labels = {         "name": "CockroachDB Operator",         "vendor": "Cockroach Labs",-        # FIXME - labels do not support a stamped variable such as {STABLE_DOCKER_TAG}-        "version": "v1.0.0-rc.0",-        "release": "0",-        "summary": "CockroachDB is a Distributed SQL database",-        "description": "CockroachDB is a PostgreSQL wire-compatible DistributedSQL database",+        "version": "{STABLE_DOCKER_TAG}",+        "release": "{STABLE_DOCKER_TAG}",

show-current is not working on debian

chrislovecnm

comment created time in 14 hours

Pull request review commentcockroachdb/cockroach-operator

Fixing stamping in container

 container_image(     labels = {         "name": "CockroachDB Operator",         "vendor": "Cockroach Labs",-        # FIXME - labels do not support a stamped variable such as {STABLE_DOCKER_TAG}-        "version": "v1.0.0-rc.0",-        "release": "0",-        "summary": "CockroachDB is a Distributed SQL database",-        "description": "CockroachDB is a PostgreSQL wire-compatible DistributedSQL database",+        "version": "{STABLE_DOCKER_TAG}",+        "release": "{STABLE_DOCKER_TAG}",

git rev-list $(git branch --show-current) | wc -l

chrislovecnm

comment created time in 15 hours

Pull request review commentcockroachdb/cockroach-operator

Fixing stamping in container

 container_image(     labels = {         "name": "CockroachDB Operator",         "vendor": "Cockroach Labs",-        # FIXME - labels do not support a stamped variable such as {STABLE_DOCKER_TAG}-        "version": "v1.0.0-rc.0",-        "release": "0",-        "summary": "CockroachDB is a Distributed SQL database",-        "description": "CockroachDB is a PostgreSQL wire-compatible DistributedSQL database",+        "version": "{STABLE_DOCKER_TAG}",+        "release": "{STABLE_DOCKER_TAG}",

I'm curious why not use the number of commits since the last tag on the current branch? Assuming the tag is already made and the tag is for the current commit, it would usually result in 0 (since there are 0 commits since the current tag). And then, if it's not tagged, it would show the number of commits since whatever the last tag was.

What about one of these:

  • Number of commits since the major release was tagged:
    git describe --tags  --match=[TAG_NAME] 2> /dev/null | sed -e 's/^.*-\([0-9]*\)-g[0-9a-f]*$/\1/'
    
  • Total number of commits on the current branch:
    git rev-list $(git branch --show-current) | wc -l
    

The numbers for all commits on master or all commits on all branches are not stable (they'll change with every commit merged to master / every commit added to the repo).

chrislovecnm

comment created time in 15 hours

pull request commentjonhoo/flurry

Move epoch garbage collection to flize

It's done by passing -Zbuild-std to cargo.

domenicquirl

comment created time in 15 hours

pull request commentjonhoo/flurry

Move epoch garbage collection to flize

Hm, this would be a separate -Z flag, right? I don't see that in the CI commands, but I don't actually know. @jonhoo do you?

domenicquirl

comment created time in 15 hours

pull request commentjonhoo/flurry

Move epoch garbage collection to flize

I've also added accelerate barriers for macOS now which may be of interest.

domenicquirl

comment created time in 15 hours

pull request commentjonhoo/flurry

Move epoch garbage collection to flize

The miri failure can be fixed in testing by disabling the fast-barrier feature so it doesn't attempt to use the accelerated OS barriers strategy. I'll take a look at ASAN/LSAN but are you recompiling std when getting those warnings?

domenicquirl

comment created time in 15 hours

issue commentcockroachdb/cockroach-operator

Kubernetes Operator Telemetry

@chrislovecnm , yes, I see that is implemented.

To confirm, is there a way that can we work around this policy?

In addition, based on this environment variable, there is no way to distinguish between OpenShift clusters and non-OpenShift clusters? We need to be able to see the difference in our telemetry.

johnrk

comment created time in 17 hours

issue commentcockroachdb/cockroach-operator

Test decommission feature and partial updates

@alinadonisa , is this a request for me to acceptance test the decommissioning improvements that you made?

alinadonisa

comment created time in 17 hours

delete branch mikkeloscar/pdb-controller

delete branch : pods-rbac

delete time in 17 hours

push eventmikkeloscar/pdb-controller

Mikkel Oscar Lyderik Larsen

commit sha 7416dc3fe6f5ecbcc45a5a5c17da4c979bfb0992

Add permissions to read pods Signed-off-by: Mikkel Oscar Lyderik Larsen <mikkel.larsen@zalando.de>

view details

Mikkel Oscar Lyderik Larsen

commit sha 239d52c215ffc6e7d1264a6e717e001d542d25c3

Merge pull request #40 from mikkeloscar/pods-rbac Add permissions to read pods

view details

push time in 17 hours

PR merged mikkeloscar/pdb-controller

Add permissions to read pods

The controller needs to read pods, add permissions.

Fix #39

+1 -1

3 comments

1 changed file

mikkeloscar

pr closed time in 17 hours

issue closedmikkeloscar/pdb-controller

pdb controller keeps logging "Failed to get pod lastTransitionTime: pods is forbidden"

My pdb controller keeps logging

time="2020-11-23T10:43:31Z" level=error msg="Failed to get pod lastTransitionTime: pods is forbidden: User \"system:serviceaccount:kube-system:pdb-controller\" cannot list resource \"pods\" in API group \"\" in the namespace \"<namespace>\""

Is this something that needs to be added to the example RBAC in https://github.com/mikkeloscar/pdb-controller/blob/master/docs/rbac.yaml ? If so, i'll be happy to help out with a MR addressing this issue. Otherwise, am I missing something here?

closed time in 17 hours

TheKangaroo

pull request commentmikkeloscar/pdb-controller

Add permissions to read pods

:+1:

mikkeloscar

comment created time in 17 hours

pull request commentmikkeloscar/pdb-controller

Add permissions to read pods

:+1:

mikkeloscar

comment created time in 17 hours

pull request commentmikkeloscar/pdb-controller

Add permissions to read pods

👍

mikkeloscar

comment created time in 17 hours

pull request commentmikkeloscar/pdb-controller

Add permissions to read pods

:+1:

mikkeloscar

comment created time in 18 hours

PR opened mikkeloscar/pdb-controller

Add permissions to read pods

The controller needs to read pods, add permissions.

Fix #39

+1 -1

0 comment

1 changed file

pr created time in 18 hours

create barnchmikkeloscar/pdb-controller

branch : pods-rbac

created branch time in 18 hours

issue openedcockroachdb/cockroach-operator

Add option to define additonal custom labels and annotations

Add an Option to define custom labels and annotations to the CRD, which are then applied, to the StatefulSet as well as the Pods

created time in 18 hours

issue openedmikkeloscar/pdb-controller

pdb controller keeps logging "Failed to get pod lastTransitionTime: pods is forbidden"

My pdb controller keeps logging

time="2020-11-23T10:43:31Z" level=error msg="Failed to get pod lastTransitionTime: pods is forbidden: User \"system:serviceaccount:kube-system:pdb-controller\" cannot list resource \"pods\" in API group \"\" in the namespace \"<namespace>\""

Is this something that needs to be added to the example RBAC in https://github.com/mikkeloscar/pdb-controller/blob/master/docs/rbac.yaml ? If so, i'll be happy to help out with a MR addressing this issue. Otherwise, am I missing something here?

created time in 20 hours

startedsigp/lighthouse

started time in a day

startedPostgREST/postgrest

started time in a day

fork brushmate/esphome-1

ESPHome is a system to control your ESP8266/ESP32 by simple yet powerful configuration files and control them remotely through Home Automation systems.

https://esphome.io/

fork in 2 days

startedrust-lang/log

started time in 2 days

more