GOPATH set to /go
rm -rf jemalloc.src
./scripts/untarxz.sh /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src.tar.xz -C /go/src/github.com/cockroachdb/cockroach/c-deps
touch /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/.extracted
rm -rf /go/native/x86_64-pc-linux-gnu/jemalloc
mkdir -p /go/native/x86_64-pc-linux-gnu/jemalloc
cd /go/native/x86_64-pc-linux-gnu/jemalloc && /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/configure  --enable-prof
checking for xsltproc... false
checking for gcc... no
checking for cc... cc
checking whether the C compiler works... yes
checking for C compiler default output file name... a.out
checking for suffix of executables...
checking whether we are cross compiling... no
checking for suffix of object files... o
checking whether we are using the GNU C compiler... yes
checking whether cc accepts -g... yes
checking for cc option to accept ISO C89... none needed
checking whether compiler is cray... no
checking whether compiler supports -std=gnu11... yes
checking whether compiler supports -Wall... yes
checking whether compiler supports -Werror=declaration-after-statement... yes
checking whether compiler supports -Wshorten-64-to-32... yes
checking whether compiler supports -Wsign-compare... yes
checking whether compiler supports -pipe... yes
checking whether compiler supports -g3... yes
checking how to run the C preprocessor... cc -E
checking for grep that handles long lines and -e... /bin/grep
checking for egrep... /bin/grep -E
checking for ANSI C header files... yes
checking for sys/types.h... yes
checking for sys/stat.h... yes
checking for stdlib.h... yes
checking for string.h... yes
checking for memory.h... yes
checking for strings.h... yes
checking for inttypes.h... yes
checking for stdint.h... yes
checking for unistd.h... yes
checking whether byte ordering is bigendian... no
checking size of void *... 8
checking size of int... 4
checking size of long... 8
checking size of long long... 8
checking size of intmax_t... 8
checking build system type... x86_64-pc-linux-gnu
checking host system type... x86_64-pc-linux-gnu
checking whether pause instruction is compilable... yes
checking for ar... ar
checking malloc.h usability... yes
checking malloc.h presence... yes
checking for malloc.h... yes
checking whether malloc_usable_size definition can use const argument... no
checking for library containing log... -lm
checking whether __attribute__ syntax is compilable... yes
checking whether compiler supports -fvisibility=hidden... yes
checking whether compiler supports -Werror... yes
checking whether compiler supports -herror_on_warning... no
checking whether tls_model attribute is compilable... yes
checking whether compiler supports -Werror... yes
checking whether compiler supports -herror_on_warning... no
checking whether alloc_size attribute is compilable... no
checking whether compiler supports -Werror... yes
checking whether compiler supports -herror_on_warning... no
checking whether format(gnu_printf, ...) attribute is compilable... no
checking whether compiler supports -Werror... yes
checking whether compiler supports -herror_on_warning... no
checking whether format(printf, ...) attribute is compilable... yes
checking for a BSD-compatible install... /usr/bin/install -c
checking for ranlib... ranlib
checking for ld... /usr/bin/ld
checking for autoconf... /usr/bin/autoconf
checking for memalign... yes
checking for valloc... yes
checking whether compiler supports -O3... yes
checking whether compiler supports -funroll-loops... yes
checking unwind.h usability... yes
checking unwind.h presence... yes
checking for unwind.h... yes
checking for _Unwind_Backtrace in -lgcc... yes
checking configured backtracing method... libgcc
checking for sbrk... yes
checking whether utrace(2) is compilable... no
checking whether valgrind is compilable... no
checking whether a program using __builtin_unreachable is compilable... yes
checking whether a program using __builtin_ffsl is compilable... yes
checking LG_PAGE... 12
checking pthread.h usability... yes
checking pthread.h presence... yes
checking for pthread.h... yes
checking for pthread_create in -lpthread... yes
checking whether pthread_atfork(3) is compilable... yes
checking for library containing clock_gettime... none required
checking whether clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is compilable... yes
checking whether clock_gettime(CLOCK_MONOTONIC, ...) is compilable... yes
checking whether mach_absolute_time() is compilable... no
checking whether compiler supports -Werror... yes
checking whether syscall(2) is compilable... yes
checking for secure_getenv... yes
checking for issetugid... no
checking for _malloc_thread_cleanup... no
checking for _pthread_mutex_init_calloc_cb... no
checking for TLS... yes
checking whether C11 atomics is compilable... yes
checking whether atomic(9) is compilable... no
checking whether Darwin OSAtomic*() is compilable... no
checking whether madvise(2) is compilable... yes
checking whether madvise(..., MADV_FREE) is compilable... no
checking whether madvise(..., MADV_DONTNEED) is compilable... yes
checking whether madvise(..., MADV_[NO]HUGEPAGE) is compilable... yes
checking whether to force 32-bit __sync_{add,sub}_and_fetch()... no
checking whether to force 64-bit __sync_{add,sub}_and_fetch()... no
checking for __builtin_clz... yes
checking whether Darwin os_unfair_lock_*() is compilable... no
checking whether Darwin OSSpin*() is compilable... no
checking whether glibc malloc hook is compilable... yes
checking whether glibc memalign hook is compilable... yes
checking whether pthreads adaptive mutexes is compilable... yes
checking for stdbool.h that conforms to C99... yes
checking for _Bool... yes
configure: creating ./config.status
config.status: creating Makefile
config.status: creating jemalloc.pc
config.status: creating doc/html.xsl
config.status: creating doc/manpages.xsl
config.status: creating doc/jemalloc.xml
config.status: creating include/jemalloc/jemalloc_macros.h
config.status: creating include/jemalloc/jemalloc_protos.h
config.status: creating include/jemalloc/jemalloc_typedefs.h
config.status: creating include/jemalloc/internal/jemalloc_internal.h
config.status: creating test/test.sh
config.status: creating test/include/test/jemalloc_test.h
config.status: creating config.stamp
config.status: creating bin/jemalloc-config
config.status: creating bin/jemalloc.sh
config.status: creating bin/jeprof
config.status: creating include/jemalloc/jemalloc_defs.h
config.status: creating include/jemalloc/internal/jemalloc_internal_defs.h
config.status: creating test/include/test/jemalloc_test_defs.h
config.status: executing include/jemalloc/internal/private_namespace.h commands
config.status: executing include/jemalloc/internal/private_unnamespace.h commands
config.status: executing include/jemalloc/internal/public_symbols.txt commands
config.status: executing include/jemalloc/internal/public_namespace.h commands
config.status: executing include/jemalloc/internal/public_unnamespace.h commands
config.status: executing include/jemalloc/internal/size_classes.h commands
config.status: executing include/jemalloc/jemalloc_protos_jet.h commands
config.status: executing include/jemalloc/jemalloc_rename.h commands
config.status: executing include/jemalloc/jemalloc_mangle.h commands
config.status: executing include/jemalloc/jemalloc_mangle_jet.h commands
config.status: executing include/jemalloc/jemalloc.h commands
===============================================================================
jemalloc version   : 4.5.0-0-g04380e79f1e2428bd0ad000bbc6e3d2dfc6b66a5
library revision   : 2

CONFIG             : --enable-prof
CC                 : cc
CONFIGURE_CFLAGS   : -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops
SPECIFIED_CFLAGS   :
EXTRA_CFLAGS       :
CPPFLAGS           : -D_GNU_SOURCE -D_REENTRANT
LDFLAGS            :
EXTRA_LDFLAGS      :
LIBS               : -lm  -lgcc -lm -lpthread
RPATH_EXTRA        :

XSLTPROC           : false
XSLROOT            :

PREFIX             : /usr/local
BINDIR             : /usr/local/bin
DATADIR            : /usr/local/share
INCLUDEDIR         : /usr/local/include
LIBDIR             : /usr/local/lib
MANDIR             : /usr/local/share/man

srcroot            : /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/
abs_srcroot        : /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/
objroot            :
abs_objroot        : /go/native/x86_64-pc-linux-gnu/jemalloc/

JEMALLOC_PREFIX    :
JEMALLOC_PRIVATE_NAMESPACE
                   : je_
install_suffix     :
malloc_conf        :
autogen            : 0
cc-silence         : 1
debug              : 0
code-coverage      : 0
stats              : 1
prof               : 1
prof-libunwind     : 0
prof-libgcc        : 1
prof-gcc           : 0
tcache             : 1
thp                : 1
fill               : 1
utrace             : 0
valgrind           : 0
xmalloc            : 0
munmap             : 0
lazy_lock          : 0
tls                : 1
cache-oblivious    : 1
===============================================================================
git submodule update --init
github.com/cockroachdb/cockroach/pkg/cmd/metacheck
github.com/cockroachdb/cockroach/pkg/cmd/ncpus
github.com/cockroachdb/cockroach/pkg/cmd/returncheck
github.com/cockroachdb/cockroach/vendor/github.com/client9/misspell/cmd/misspell
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/crlfmt
github.com/cockroachdb/cockroach/vendor/github.com/Masterminds/glide
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/stress
github.com/cockroachdb/cockroach/vendor/github.com/golang/lint/golint
github.com/cockroachdb/cockroach/vendor/github.com/google/pprof
github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
github.com/cockroachdb/cockroach/vendor/github.com/jteeuwen/go-bindata/go-bindata
github.com/cockroachdb/cockroach/vendor/github.com/kisielk/errcheck
github.com/cockroachdb/cockroach/vendor/github.com/mattn/goveralls
github.com/cockroachdb/cockroach/vendor/github.com/mdempsky/unconvert
github.com/cockroachdb/cockroach/vendor/github.com/mibk/dupl
github.com/cockroachdb/cockroach/vendor/github.com/wadey/gocovmerge
github.com/cockroachdb/cockroach/vendor/golang.org/x/perf/cmd/benchstat
github.com/cockroachdb/cockroach/vendor/golang.org/x/tools/cmd/goimports
github.com/cockroachdb/cockroach/vendor/golang.org/x/tools/cmd/gorename
github.com/cockroachdb/cockroach/vendor/golang.org/x/tools/cmd/goyacc
github.com/cockroachdb/cockroach/vendor/golang.org/x/tools/cmd/guru
github.com/cockroachdb/cockroach/vendor/golang.org/x/tools/cmd/stringer
touch /go/src/github.com/cockroachdb/cockroach/bin/.bootstrap
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/jemalloc.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/jemalloc.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/arena.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/arena.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/base.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/base.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/bitmap.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/bitmap.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/chunk.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/chunk.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/chunk_dss.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/chunk_dss.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/atomic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/atomic.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/chunk_mmap.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/chunk_mmap.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/ckh.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/ckh.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/ctl.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/ctl.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/extent.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/extent.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/hash.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/hash.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/huge.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/huge.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/mb.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/mb.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/mutex.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/mutex.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/nstime.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/nstime.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/pages.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/pages.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/prng.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/prng.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/prof.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/prof.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/quarantine.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/quarantine.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/rtree.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/rtree.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/stats.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/stats.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/spin.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/spin.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/tcache.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/tcache.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/ticker.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/ticker.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/tsd.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/tsd.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/util.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/util.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/witness.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/witness.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/jemalloc.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/jemalloc.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/arena.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/arena.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/atomic.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/atomic.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/bitmap.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/bitmap.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/base.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/base.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/chunk.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/chunk.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/chunk_dss.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/chunk_dss.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/chunk_mmap.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/chunk_mmap.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/ckh.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/ckh.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/ctl.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/ctl.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/extent.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/extent.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/hash.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/hash.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/huge.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/huge.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/mb.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/mb.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/mutex.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/mutex.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/nstime.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/nstime.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/pages.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/pages.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/prng.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/prng.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/prof.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/prof.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/quarantine.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/quarantine.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/rtree.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/rtree.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/stats.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/stats.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/spin.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/spin.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/tcache.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/tcache.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/ticker.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/ticker.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/tsd.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/tsd.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/util.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/util.c
cc -std=gnu11 -Wall -Werror=declaration-after-statement -Wshorten-64-to-32 -Wsign-compare -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -fPIC -DPIC -c -D_GNU_SOURCE -D_REENTRANT -I/go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/include -Iinclude -o src/witness.pic.o /go/src/github.com/cockroachdb/cockroach/c-deps/jemalloc.src/src/witness.c
ar crus lib/libjemalloc.a src/jemalloc.o src/arena.o src/atomic.o src/base.o src/bitmap.o src/chunk.o src/chunk_dss.o src/chunk_mmap.o src/ckh.o src/ctl.o src/extent.o src/hash.o src/huge.o src/mb.o src/mutex.o src/nstime.o src/pages.o src/prng.o src/prof.o src/quarantine.o src/rtree.o src/stats.o src/spin.o src/tcache.o src/ticker.o src/tsd.o src/util.o src/witness.o
ar: `u' modifier ignored since `D' is the default (see `U')
ar crus lib/libjemalloc_pic.a src/jemalloc.pic.o src/arena.pic.o src/atomic.pic.o src/base.pic.o src/bitmap.pic.o src/chunk.pic.o src/chunk_dss.pic.o src/chunk_mmap.pic.o src/ckh.pic.o src/ctl.pic.o src/extent.pic.o src/hash.pic.o src/huge.pic.o src/mb.pic.o src/mutex.pic.o src/nstime.pic.o src/pages.pic.o src/prng.pic.o src/prof.pic.o src/quarantine.pic.o src/rtree.pic.o src/stats.pic.o src/spin.pic.o src/tcache.pic.o src/ticker.pic.o src/tsd.pic.o src/util.pic.o src/witness.pic.o
ar: `u' modifier ignored since `D' is the default (see `U')
rm -rf protobuf.src
./scripts/untarxz.sh /go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src.tar.xz -C /go/src/github.com/cockroachdb/cockroach/c-deps
touch /go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/.extracted
rm -rf /go/native/x86_64-pc-linux-gnu/protobuf
mkdir -p /go/native/x86_64-pc-linux-gnu/protobuf
cd /go/native/x86_64-pc-linux-gnu/protobuf && cmake  -Dprotobuf_BUILD_TESTS=OFF /go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/cmake
-- The C compiler identification is Clang 3.8.0
-- The CXX compiler identification is Clang 3.8.0
-- Check for working C compiler: /usr/bin/cc
-- Check for working C compiler: /usr/bin/cc -- works
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Detecting C compile features
-- Detecting C compile features - done
-- Check for working CXX compiler: /usr/bin/c++
-- Check for working CXX compiler: /usr/bin/c++ -- works
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Looking for pthread.h
-- Looking for pthread.h - found
-- Looking for pthread_create
-- Looking for pthread_create - not found
-- Looking for pthread_create in pthreads
-- Looking for pthread_create in pthreads - not found
-- Looking for pthread_create in pthread
-- Looking for pthread_create in pthread - found
-- Found Threads: TRUE
-- Could NOT find ZLIB (missing:  ZLIB_LIBRARY ZLIB_INCLUDE_DIR)
-- Configuring done
-- Generating done
-- Build files have been written to: /go/native/x86_64-pc-linux-gnu/protobuf
Scanning dependencies of target libprotobuf
[  2%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/arena.cc.o
[  2%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/arenastring.cc.o
[  5%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/extension_set.cc.o
[  5%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/io/zero_copy_stream.cc.o
[  5%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/generated_message_util.cc.o
[  7%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/io/zero_copy_stream_impl_lite.cc.o
[  7%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/message_lite.cc.o
[ 10%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/io/coded_stream.cc.o
[ 12%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/repeated_field.cc.o
[ 12%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/stubs/atomicops_internals_x86_gcc.cc.o
[ 15%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/stubs/atomicops_internals_x86_msvc.cc.o
[ 15%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/stubs/bytestream.cc.o
[ 17%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/stubs/common.cc.o
[ 17%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/stubs/int128.cc.o
[ 20%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/stubs/once.cc.o
[ 20%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/stubs/status.cc.o
[ 23%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/stubs/statusor.cc.o
[ 23%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/stubs/stringpiece.cc.o
[ 25%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/stubs/stringprintf.cc.o
[ 25%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/stubs/structurally_valid.cc.o
[ 28%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/stubs/strutil.cc.o
[ 28%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/stubs/time.cc.o
[ 30%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/wire_format_lite.cc.o
[ 30%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/any.cc.o
[ 33%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/any.pb.cc.o
[ 33%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/api.pb.cc.o
[ 35%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/compiler/importer.cc.o
[ 35%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/compiler/parser.cc.o
[ 38%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/descriptor.cc.o
[ 38%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/descriptor.pb.cc.o
[ 41%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/descriptor_database.cc.o
[ 41%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/duration.pb.cc.o
[ 43%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/dynamic_message.cc.o
[ 43%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/empty.pb.cc.o
[ 46%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/extension_set_heavy.cc.o
[ 46%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/field_mask.pb.cc.o
[ 48%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/generated_message_reflection.cc.o
[ 48%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/io/gzip_stream.cc.o
[ 51%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/io/printer.cc.o
[ 51%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/io/strtod.cc.o
[ 53%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/io/tokenizer.cc.o
[ 53%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/io/zero_copy_stream_impl.cc.o
[ 56%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/map_field.cc.o
[ 56%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/message.cc.o
[ 58%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/reflection_ops.cc.o
[ 58%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/service.cc.o
[ 61%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/source_context.pb.cc.o
[ 61%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/struct.pb.cc.o
[ 64%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/stubs/mathlimits.cc.o
[ 64%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/stubs/substitute.cc.o
[ 66%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/text_format.cc.o
[ 66%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/timestamp.pb.cc.o
[ 69%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/type.pb.cc.o
[ 69%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/unknown_field_set.cc.o
[ 71%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/delimited_message_util.cc.o
[ 71%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/field_comparator.cc.o
[ 74%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/field_mask_util.cc.o
[ 74%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/internal/datapiece.cc.o
[ 76%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/internal/default_value_objectwriter.cc.o
[ 76%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/internal/error_listener.cc.o
[ 79%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/internal/field_mask_utility.cc.o
[ 79%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/internal/json_escaping.cc.o
[ 82%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/internal/json_objectwriter.cc.o
[ 82%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/internal/json_stream_parser.cc.o
[ 84%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/internal/object_writer.cc.o
[ 84%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/internal/proto_writer.cc.o
[ 87%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/internal/protostream_objectsource.cc.o
[ 87%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/internal/protostream_objectwriter.cc.o
[ 89%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/internal/type_info.cc.o
[ 89%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/internal/type_info_test_helper.cc.o
[ 92%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/internal/utility.cc.o
[ 92%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/json_util.cc.o
[ 94%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/message_differencer.cc.o
[ 94%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/time_util.cc.o
[ 97%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/util/type_resolver_util.cc.o
[ 97%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/wire_format.cc.o
[100%] Building CXX object CMakeFiles/libprotobuf.dir/go/src/github.com/cockroachdb/cockroach/c-deps/protobuf.src/src/google/protobuf/wrappers.pb.cc.o
[100%] Linking CXX static library libprotobuf.a
[100%] Built target libprotobuf
rm -rf snappy.src
./scripts/untarxz.sh /go/src/github.com/cockroachdb/cockroach/c-deps/snappy.src.tar.xz -C /go/src/github.com/cockroachdb/cockroach/c-deps
touch /go/src/github.com/cockroachdb/cockroach/c-deps/snappy.src/.extracted
mkdir -p /go/native/x86_64-pc-linux-gnu/snappy
cd /go/native/x86_64-pc-linux-gnu/snappy && /go/src/github.com/cockroachdb/cockroach/c-deps/snappy.src/configure  --disable-shared --disable-gtest
checking for a BSD-compatible install... /usr/bin/install -c
checking whether build environment is sane... yes
checking for a thread-safe mkdir -p... /bin/mkdir -p
checking for gawk... gawk
checking whether make sets $(MAKE)... yes
checking whether make supports nested variables... yes
checking build system type... x86_64-unknown-linux-gnu
checking host system type... x86_64-unknown-linux-gnu
checking how to print strings... printf
checking for style of include used by make... GNU
checking for gcc... no
checking for cc... cc
checking whether the C compiler works... yes
checking for C compiler default output file name... a.out
checking for suffix of executables...
checking whether we are cross compiling... no
checking for suffix of object files... o
checking whether we are using the GNU C compiler... yes
checking whether cc accepts -g... yes
checking for cc option to accept ISO C89... none needed
checking whether cc understands -c and -o together... yes
checking dependency style of cc... gcc3
checking for a sed that does not truncate output... /bin/sed
checking for grep that handles long lines and -e... /bin/grep
checking for egrep... /bin/grep -E
checking for fgrep... /bin/grep -F
checking for ld used by cc... /usr/bin/ld
checking if the linker (/usr/bin/ld) is GNU ld... yes
checking for BSD- or MS-compatible name lister (nm)... /usr/bin/nm -B
checking the name lister (/usr/bin/nm -B) interface... BSD nm
checking whether ln -s works... yes
checking the maximum length of command line arguments... 1572864
checking whether the shell understands some XSI constructs... yes
checking whether the shell understands "+="... yes
checking how to convert x86_64-unknown-linux-gnu file names to x86_64-unknown-linux-gnu format... func_convert_file_noop
checking how to convert x86_64-unknown-linux-gnu file names to toolchain format... func_convert_file_noop
checking for /usr/bin/ld option to reload object files... -r
checking for objdump... objdump
checking how to recognize dependent libraries... pass_all
checking for dlltool... no
checking how to associate runtime and link libraries... printf %s\n
checking for ar... ar
checking for archiver @FILE support... @
checking for strip... strip
checking for ranlib... ranlib
checking command to parse /usr/bin/nm -B output from cc object... ok
checking for sysroot... no
checking for mt... no
checking if : is a manifest tool... no
checking how to run the C preprocessor... cc -E
checking for ANSI C header files... yes
checking for sys/types.h... yes
checking for sys/stat.h... yes
checking for stdlib.h... yes
checking for string.h... yes
checking for memory.h... yes
checking for strings.h... yes
checking for inttypes.h... yes
checking for stdint.h... yes
checking for unistd.h... yes
checking for dlfcn.h... yes
checking for objdir... .libs
checking if cc supports -fno-rtti -fno-exceptions... yes
checking for cc option to produce PIC... -fPIC -DPIC
checking if cc PIC flag -fPIC -DPIC works... yes
checking if cc static flag -static works... yes
checking if cc supports -c -o file.o... yes
checking if cc supports -c -o file.o... (cached) yes
checking whether the cc linker (/usr/bin/ld -m elf_x86_64) supports shared libraries... yes
checking dynamic linker characteristics... GNU/Linux ld.so
checking how to hardcode library paths into programs... immediate
checking whether stripping libraries is possible... yes
checking if libtool supports shared libraries... yes
checking whether to build shared libraries... no
checking whether to build static libraries... yes
checking for g++... no
checking for c++... c++
checking whether we are using the GNU C++ compiler... yes
checking whether c++ accepts -g... yes
checking dependency style of c++... gcc3
checking how to run the C++ preprocessor... c++ -E
checking for ld used by c++... /usr/bin/ld -m elf_x86_64
checking if the linker (/usr/bin/ld -m elf_x86_64) is GNU ld... yes
checking whether the c++ linker (/usr/bin/ld -m elf_x86_64) supports shared libraries... yes
checking for c++ option to produce PIC... -fPIC -DPIC
checking if c++ PIC flag -fPIC -DPIC works... yes
checking if c++ static flag -static works... yes
checking if c++ supports -c -o file.o... yes
checking if c++ supports -c -o file.o... (cached) yes
checking whether the c++ linker (/usr/bin/ld -m elf_x86_64) supports shared libraries... yes
checking dynamic linker characteristics... (cached) GNU/Linux ld.so
checking how to hardcode library paths into programs... immediate
checking whether byte ordering is bigendian... no
checking for size_t... yes
checking for ssize_t... yes
checking for stdint.h... (cached) yes
checking stddef.h usability... yes
checking stddef.h presence... yes
checking for stddef.h... yes
checking sys/mman.h usability... yes
checking sys/mman.h presence... yes
checking for sys/mman.h... yes
checking sys/resource.h usability... yes
checking sys/resource.h presence... yes
checking for sys/resource.h... yes
checking windows.h usability... no
checking windows.h presence... no
checking for windows.h... no
checking byteswap.h usability... yes
checking byteswap.h presence... yes
checking for byteswap.h... yes
checking sys/byteswap.h usability... no
checking sys/byteswap.h presence... no
checking for sys/byteswap.h... no
checking sys/endian.h usability... no
checking sys/endian.h presence... no
checking for sys/endian.h... no
checking sys/time.h usability... yes
checking sys/time.h presence... yes
checking for sys/time.h... yes
checking for mmap... yes
checking for pkg-config... no
checking for gflags... no
checking if the compiler supports __builtin_expect... yes
checking if the compiler supports __builtin_ctzll... yes
checking for zlibVersion in -lz... no
checking for lzo1x_1_15_compress in -llzo2... no
checking for lzf_compress in -llzf... no
checking for fastlz_compress in -lfastlz... no
checking for qlz_compress in -lquicklz... no
checking that generated files are newer than configure... done
configure: creating ./config.status
config.status: creating Makefile
config.status: creating snappy-stubs-public.h
config.status: creating config.h
config.status: config.h is unchanged
config.status: executing depfiles commands
config.status: executing libtool commands
make  all-am
rm -rf rocksdb.src
./scripts/untarxz.sh /go/src/github.com/cockroachdb/cockroach/c-deps/rocksdb.src.tar.xz -C /go/src/github.com/cockroachdb/cockroach/c-deps
touch /go/src/github.com/cockroachdb/cockroach/c-deps/rocksdb.src/.extracted
rm -rf /go/native/x86_64-pc-linux-gnu/rocksdb
mkdir -p /go/native/x86_64-pc-linux-gnu/rocksdb
cd /go/native/x86_64-pc-linux-gnu/rocksdb && cmake  /go/src/github.com/cockroachdb/cockroach/c-deps/rocksdb.src \
  -DWITH_SSE42=OFF \
  -DSNAPPY_LIBRARIES=/go/native/x86_64-pc-linux-gnu/snappy/.libs/libsnappy.a -DSNAPPY_INCLUDE_DIR=/go/src/github.com/cockroachdb/cockroach/c-deps/snappy.src -DWITH_SNAPPY=ON \
  -DJEMALLOC_LIBRARIES=/go/native/x86_64-pc-linux-gnu/jemalloc/lib/libjemalloc.a -DJEMALLOC_INCLUDE_DIR=/go/native/x86_64-pc-linux-gnu/jemalloc/include -DWITH_JEMALLOC=ON \
  -DCMAKE_CXX_FLAGS=-DNDEBUG
-- The C compiler identification is Clang 3.8.0
-- The CXX compiler identification is Clang 3.8.0
-- Check for working C compiler: /usr/bin/cc
-- Check for working C compiler: /usr/bin/cc -- works
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Detecting C compile features
-- Detecting C compile features - done
-- Check for working CXX compiler: /usr/bin/c++
-- Check for working CXX compiler: /usr/bin/c++ -- works
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Found jemalloc: /go/native/x86_64-pc-linux-gnu/jemalloc/lib/libjemalloc.a
-- Found snappy: /go/native/x86_64-pc-linux-gnu/snappy/.libs/libsnappy.a
-- Found Git: /usr/bin/git (found version "2.7.4")
-- Performing Test HAVE_OMIT_LEAF_FRAME_POINTER
-- Performing Test HAVE_OMIT_LEAF_FRAME_POINTER - Success
-- Performing Test HAVE_FALLOCATE
-- Performing Test HAVE_FALLOCATE - Success
-- Looking for malloc_usable_size
-- Looking for malloc_usable_size - found
-- Looking for pthread.h
-- Looking for pthread.h - found
-- Looking for pthread_create
-- Looking for pthread_create - not found
-- Looking for pthread_create in pthreads
-- Looking for pthread_create in pthreads - not found
-- Looking for pthread_create in pthread
-- Looking for pthread_create in pthread - found
-- Found Threads: TRUE
-- JNI library is disabled
-- Configuring done
-- Generating done
-- Build files have been written to: /go/native/x86_64-pc-linux-gnu/rocksdb
Scanning dependencies of target build_version
[  0%] Building CXX object CMakeFiles/build_version.dir/build_version.cc.o
[  0%] Built target build_version
Scanning dependencies of target rocksdb
[  0%] Building CXX object CMakeFiles/rocksdb.dir/db/auto_roll_logger.cc.o
[  0%] Building CXX object CMakeFiles/rocksdb.dir/db/c.cc.o
[  3%] Building CXX object CMakeFiles/rocksdb.dir/db/builder.cc.o
[  3%] Building CXX object CMakeFiles/rocksdb.dir/db/column_family.cc.o
[  3%] Building CXX object CMakeFiles/rocksdb.dir/db/compaction.cc.o
[  3%] Building CXX object CMakeFiles/rocksdb.dir/db/compaction_job.cc.o
[  3%] Building CXX object CMakeFiles/rocksdb.dir/db/compacted_db_impl.cc.o
[  3%] Building CXX object CMakeFiles/rocksdb.dir/db/compaction_iterator.cc.o
[  6%] Building CXX object CMakeFiles/rocksdb.dir/db/compaction_picker.cc.o
[  6%] Building CXX object CMakeFiles/rocksdb.dir/db/convenience.cc.o
[  6%] Building CXX object CMakeFiles/rocksdb.dir/db/dbformat.cc.o
[  6%] Building CXX object CMakeFiles/rocksdb.dir/db/db_filesnapshot.cc.o
[  6%] Building CXX object CMakeFiles/rocksdb.dir/db/db_impl.cc.o
[  6%] Building CXX object CMakeFiles/rocksdb.dir/db/db_impl_debug.cc.o
[ 10%] Building CXX object CMakeFiles/rocksdb.dir/db/db_impl_experimental.cc.o
[ 10%] Building CXX object CMakeFiles/rocksdb.dir/db/db_impl_readonly.cc.o
[ 10%] Building CXX object CMakeFiles/rocksdb.dir/db/db_info_dumper.cc.o
[ 10%] Building CXX object CMakeFiles/rocksdb.dir/db/db_iter.cc.o
[ 10%] Building CXX object CMakeFiles/rocksdb.dir/db/event_helpers.cc.o
[ 10%] Building CXX object CMakeFiles/rocksdb.dir/db/external_sst_file_ingestion_job.cc.o
[ 10%] Building CXX object CMakeFiles/rocksdb.dir/db/experimental.cc.o
[ 13%] Building CXX object CMakeFiles/rocksdb.dir/db/filename.cc.o
[ 13%] Building CXX object CMakeFiles/rocksdb.dir/db/file_indexer.cc.o
[ 13%] Building CXX object CMakeFiles/rocksdb.dir/db/flush_job.cc.o
[ 13%] Building CXX object CMakeFiles/rocksdb.dir/db/flush_scheduler.cc.o
[ 13%] Building CXX object CMakeFiles/rocksdb.dir/db/forward_iterator.cc.o
[ 13%] Building CXX object CMakeFiles/rocksdb.dir/db/internal_stats.cc.o
[ 13%] Building CXX object CMakeFiles/rocksdb.dir/db/log_reader.cc.o
[ 17%] Building CXX object CMakeFiles/rocksdb.dir/db/log_writer.cc.o
[ 17%] Building CXX object CMakeFiles/rocksdb.dir/db/malloc_stats.cc.o
[ 17%] Building CXX object CMakeFiles/rocksdb.dir/db/managed_iterator.cc.o
[ 17%] Building CXX object CMakeFiles/rocksdb.dir/db/memtable.cc.o
[ 17%] Building CXX object CMakeFiles/rocksdb.dir/db/memtable_allocator.cc.o
[ 17%] Building CXX object CMakeFiles/rocksdb.dir/db/memtable_list.cc.o
[ 20%] Building CXX object CMakeFiles/rocksdb.dir/db/merge_helper.cc.o
[ 20%] Building CXX object CMakeFiles/rocksdb.dir/db/merge_operator.cc.o
[ 20%] Building CXX object CMakeFiles/rocksdb.dir/db/range_del_aggregator.cc.o
[ 20%] Building CXX object CMakeFiles/rocksdb.dir/db/repair.cc.o
[ 20%] Building CXX object CMakeFiles/rocksdb.dir/db/snapshot_impl.cc.o
[ 20%] Building CXX object CMakeFiles/rocksdb.dir/db/table_cache.cc.o
[ 20%] Building CXX object CMakeFiles/rocksdb.dir/db/table_properties_collector.cc.o
[ 24%] Building CXX object CMakeFiles/rocksdb.dir/db/transaction_log_impl.cc.o
[ 24%] Building CXX object CMakeFiles/rocksdb.dir/db/version_builder.cc.o
[ 24%] Building CXX object CMakeFiles/rocksdb.dir/db/version_edit.cc.o
[ 24%] Building CXX object CMakeFiles/rocksdb.dir/db/version_set.cc.o
[ 24%] Building CXX object CMakeFiles/rocksdb.dir/db/wal_manager.cc.o
[ 24%] Building CXX object CMakeFiles/rocksdb.dir/db/write_batch.cc.o
[ 24%] Building CXX object CMakeFiles/rocksdb.dir/db/write_batch_base.cc.o
[ 27%] Building CXX object CMakeFiles/rocksdb.dir/db/write_controller.cc.o
[ 27%] Building CXX object CMakeFiles/rocksdb.dir/db/write_thread.cc.o
[ 27%] Building CXX object CMakeFiles/rocksdb.dir/memtable/hash_cuckoo_rep.cc.o
[ 27%] Building CXX object CMakeFiles/rocksdb.dir/memtable/hash_linklist_rep.cc.o
[ 27%] Building CXX object CMakeFiles/rocksdb.dir/memtable/hash_skiplist_rep.cc.o
[ 27%] Building CXX object CMakeFiles/rocksdb.dir/memtable/skiplistrep.cc.o
[ 31%] Building CXX object CMakeFiles/rocksdb.dir/memtable/vectorrep.cc.o
[ 31%] Building CXX object CMakeFiles/rocksdb.dir/port/stack_trace.cc.o
[ 31%] Building CXX object CMakeFiles/rocksdb.dir/table/adaptive_table_factory.cc.o
[ 31%] Building CXX object CMakeFiles/rocksdb.dir/table/block.cc.o
[ 31%] Building CXX object CMakeFiles/rocksdb.dir/table/block_based_filter_block.cc.o
[ 31%] Building CXX object CMakeFiles/rocksdb.dir/table/block_based_table_builder.cc.o
[ 31%] Building CXX object CMakeFiles/rocksdb.dir/table/block_based_table_factory.cc.o
[ 34%] Building CXX object CMakeFiles/rocksdb.dir/table/block_based_table_reader.cc.o
[ 34%] Building CXX object CMakeFiles/rocksdb.dir/table/block_builder.cc.o
[ 34%] Building CXX object CMakeFiles/rocksdb.dir/table/block_prefix_index.cc.o
[ 34%] Building CXX object CMakeFiles/rocksdb.dir/table/bloom_block.cc.o
[ 34%] Building CXX object CMakeFiles/rocksdb.dir/table/cuckoo_table_builder.cc.o
[ 34%] Building CXX object CMakeFiles/rocksdb.dir/table/cuckoo_table_factory.cc.o
[ 34%] Building CXX object CMakeFiles/rocksdb.dir/table/cuckoo_table_reader.cc.o
[ 37%] Building CXX object CMakeFiles/rocksdb.dir/table/flush_block_policy.cc.o
[ 37%] Building CXX object CMakeFiles/rocksdb.dir/table/format.cc.o
[ 37%] Building CXX object CMakeFiles/rocksdb.dir/table/full_filter_block.cc.o
[ 37%] Building CXX object CMakeFiles/rocksdb.dir/table/get_context.cc.o
[ 37%] Building CXX object CMakeFiles/rocksdb.dir/table/index_builder.cc.o
[ 37%] Building CXX object CMakeFiles/rocksdb.dir/table/iterator.cc.o
[ 41%] Building CXX object CMakeFiles/rocksdb.dir/table/merging_iterator.cc.o
[ 41%] Building CXX object CMakeFiles/rocksdb.dir/table/sst_file_writer.cc.o
[ 41%] Building CXX object CMakeFiles/rocksdb.dir/table/meta_blocks.cc.o
[ 41%] Building CXX object CMakeFiles/rocksdb.dir/table/partitioned_filter_block.cc.o
[ 41%] Building CXX object CMakeFiles/rocksdb.dir/table/plain_table_builder.cc.o
[ 41%] Building CXX object CMakeFiles/rocksdb.dir/table/plain_table_factory.cc.o
[ 41%] Building CXX object CMakeFiles/rocksdb.dir/table/plain_table_index.cc.o
[ 44%] Building CXX object CMakeFiles/rocksdb.dir/table/plain_table_key_coding.cc.o
[ 44%] Building CXX object CMakeFiles/rocksdb.dir/table/plain_table_reader.cc.o
[ 44%] Building CXX object CMakeFiles/rocksdb.dir/table/persistent_cache_helper.cc.o
[ 44%] Building CXX object CMakeFiles/rocksdb.dir/table/table_properties.cc.o
[ 44%] Building CXX object CMakeFiles/rocksdb.dir/table/two_level_iterator.cc.o
[ 44%] Building CXX object CMakeFiles/rocksdb.dir/tools/sst_dump_tool.cc.o
[ 44%] Building CXX object CMakeFiles/rocksdb.dir/tools/db_bench_tool.cc.o
[ 48%] Building CXX object CMakeFiles/rocksdb.dir/tools/dump/db_dump_tool.cc.o
[ 48%] Building CXX object CMakeFiles/rocksdb.dir/util/arena.cc.o
[ 48%] Building CXX object CMakeFiles/rocksdb.dir/util/bloom.cc.o
[ 48%] Building CXX object CMakeFiles/rocksdb.dir/util/cf_options.cc.o
[ 48%] Building CXX object CMakeFiles/rocksdb.dir/util/clock_cache.cc.o
[ 48%] Building CXX object CMakeFiles/rocksdb.dir/util/coding.cc.o
[ 51%] Building CXX object CMakeFiles/rocksdb.dir/util/compaction_job_stats_impl.cc.o
[ 51%] Building CXX object CMakeFiles/rocksdb.dir/util/comparator.cc.o
[ 51%] Building CXX object CMakeFiles/rocksdb.dir/util/concurrent_arena.cc.o
[ 51%] Building CXX object CMakeFiles/rocksdb.dir/util/crc32c.cc.o
[ 51%] Building CXX object CMakeFiles/rocksdb.dir/util/db_options.cc.o
[ 51%] Building CXX object CMakeFiles/rocksdb.dir/util/delete_scheduler.cc.o
[ 51%] Building CXX object CMakeFiles/rocksdb.dir/util/dynamic_bloom.cc.o
[ 55%] Building CXX object CMakeFiles/rocksdb.dir/util/env.cc.o
[ 55%] Building CXX object CMakeFiles/rocksdb.dir/util/env_chroot.cc.o
[ 55%] Building CXX object CMakeFiles/rocksdb.dir/util/env_hdfs.cc.o
[ 55%] Building CXX object CMakeFiles/rocksdb.dir/util/event_logger.cc.o
[ 55%] Building CXX object CMakeFiles/rocksdb.dir/util/file_util.cc.o
[ 55%] Building CXX object CMakeFiles/rocksdb.dir/util/file_reader_writer.cc.o
[ 58%] Building CXX object CMakeFiles/rocksdb.dir/util/sst_file_manager_impl.cc.o
[ 58%] Building CXX object CMakeFiles/rocksdb.dir/util/filter_policy.cc.o
[ 58%] Building CXX object CMakeFiles/rocksdb.dir/util/hash.cc.o
[ 58%] Building CXX object CMakeFiles/rocksdb.dir/util/histogram.cc.o
[ 58%] Building CXX object CMakeFiles/rocksdb.dir/util/histogram_windowing.cc.o
[ 58%] Building CXX object CMakeFiles/rocksdb.dir/util/instrumented_mutex.cc.o
[ 58%] Building CXX object CMakeFiles/rocksdb.dir/util/iostats_context.cc.o
[ 62%] Building CXX object CMakeFiles/rocksdb.dir/util/lru_cache.cc.o
[ 62%] Building CXX object CMakeFiles/rocksdb.dir/tools/ldb_cmd.cc.o
[ 62%] Building CXX object CMakeFiles/rocksdb.dir/tools/ldb_tool.cc.o
[ 62%] Building CXX object CMakeFiles/rocksdb.dir/util/logging.cc.o
[ 62%] Building CXX object CMakeFiles/rocksdb.dir/util/log_buffer.cc.o
[ 62%] Building CXX object CMakeFiles/rocksdb.dir/util/memenv.cc.o
[ 62%] Building CXX object CMakeFiles/rocksdb.dir/util/murmurhash.cc.o
[ 65%] Building CXX object CMakeFiles/rocksdb.dir/util/options.cc.o
[ 65%] Building CXX object CMakeFiles/rocksdb.dir/util/options_helper.cc.o
[ 65%] Building CXX object CMakeFiles/rocksdb.dir/util/options_parser.cc.o
[ 65%] Building CXX object CMakeFiles/rocksdb.dir/util/options_sanity_check.cc.o
[ 65%] Building CXX object CMakeFiles/rocksdb.dir/util/perf_context.cc.o
[ 65%] Building CXX object CMakeFiles/rocksdb.dir/util/perf_level.cc.o
[ 68%] Building CXX object CMakeFiles/rocksdb.dir/util/random.cc.o
[ 68%] Building CXX object CMakeFiles/rocksdb.dir/util/rate_limiter.cc.o
[ 68%] Building CXX object CMakeFiles/rocksdb.dir/util/sharded_cache.cc.o
[ 68%] Building CXX object CMakeFiles/rocksdb.dir/util/slice.cc.o
[ 68%] Building CXX object CMakeFiles/rocksdb.dir/util/statistics.cc.o
[ 68%] Building CXX object CMakeFiles/rocksdb.dir/util/status.cc.o
[ 68%] Building CXX object CMakeFiles/rocksdb.dir/util/status_message.cc.o
[ 72%] Building CXX object CMakeFiles/rocksdb.dir/util/string_util.cc.o
[ 72%] Building CXX object CMakeFiles/rocksdb.dir/util/sync_point.cc.o
[ 72%] Building CXX object CMakeFiles/rocksdb.dir/util/testutil.cc.o
[ 72%] Building CXX object CMakeFiles/rocksdb.dir/util/thread_local.cc.o
[ 72%] Building CXX object CMakeFiles/rocksdb.dir/util/threadpool_imp.cc.o
[ 72%] Building CXX object CMakeFiles/rocksdb.dir/util/thread_status_impl.cc.o
[ 72%] Building CXX object CMakeFiles/rocksdb.dir/util/thread_status_updater.cc.o
[ 75%] Building CXX object CMakeFiles/rocksdb.dir/util/thread_status_util.cc.o
[ 75%] Building CXX object CMakeFiles/rocksdb.dir/util/thread_status_util_debug.cc.o
[ 75%] Building CXX object CMakeFiles/rocksdb.dir/util/transaction_test_util.cc.o
[ 75%] Building CXX object CMakeFiles/rocksdb.dir/util/xxhash.cc.o
[ 75%] Building CXX object CMakeFiles/rocksdb.dir/utilities/backupable/backupable_db.cc.o
[ 75%] Building CXX object CMakeFiles/rocksdb.dir/utilities/blob_db/blob_db.cc.o
[ 79%] Building CXX object CMakeFiles/rocksdb.dir/utilities/checkpoint/checkpoint.cc.o
[ 79%] Building CXX object CMakeFiles/rocksdb.dir/utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc.o
[ 79%] Building CXX object CMakeFiles/rocksdb.dir/utilities/date_tiered/date_tiered_db_impl.cc.o
[ 79%] Building CXX object CMakeFiles/rocksdb.dir/utilities/document/document_db.cc.o
[ 79%] Building CXX object CMakeFiles/rocksdb.dir/utilities/document/json_document.cc.o
[ 79%] Building CXX object CMakeFiles/rocksdb.dir/utilities/document/json_document_builder.cc.o
[ 79%] Building CXX object CMakeFiles/rocksdb.dir/utilities/env_mirror.cc.o
[ 82%] Building CXX object CMakeFiles/rocksdb.dir/utilities/geodb/geodb_impl.cc.o
[ 82%] Building CXX object CMakeFiles/rocksdb.dir/utilities/leveldb_options/leveldb_options.cc.o
[ 82%] Building CXX object CMakeFiles/rocksdb.dir/utilities/lua/rocks_lua_compaction_filter.cc.o
[ 82%] Building CXX object CMakeFiles/rocksdb.dir/utilities/memory/memory_util.cc.o
[ 82%] Building CXX object CMakeFiles/rocksdb.dir/utilities/merge_operators/string_append/stringappend.cc.o
[ 82%] Building CXX object CMakeFiles/rocksdb.dir/utilities/merge_operators/string_append/stringappend2.cc.o
[ 82%] Building CXX object CMakeFiles/rocksdb.dir/utilities/merge_operators/put.cc.o
[ 86%] Building CXX object CMakeFiles/rocksdb.dir/utilities/merge_operators/max.cc.o
[ 86%] Building CXX object CMakeFiles/rocksdb.dir/utilities/merge_operators/uint64add.cc.o
[ 86%] Building CXX object CMakeFiles/rocksdb.dir/utilities/option_change_migration/option_change_migration.cc.o
[ 86%] Building CXX object CMakeFiles/rocksdb.dir/utilities/options/options_util.cc.o
[ 86%] Building CXX object CMakeFiles/rocksdb.dir/utilities/persistent_cache/block_cache_tier.cc.o
[ 86%] Building CXX object CMakeFiles/rocksdb.dir/utilities/persistent_cache/block_cache_tier_file.cc.o
[ 89%] Building CXX object CMakeFiles/rocksdb.dir/utilities/persistent_cache/block_cache_tier_metadata.cc.o
[ 89%] Building CXX object CMakeFiles/rocksdb.dir/utilities/persistent_cache/persistent_cache_tier.cc.o
[ 89%] Building CXX object CMakeFiles/rocksdb.dir/utilities/persistent_cache/volatile_tier_impl.cc.o
[ 89%] Building CXX object CMakeFiles/rocksdb.dir/utilities/redis/redis_lists.cc.o
[ 89%] Building CXX object CMakeFiles/rocksdb.dir/utilities/simulator_cache/sim_cache.cc.o
[ 89%] Building CXX object CMakeFiles/rocksdb.dir/utilities/spatialdb/spatial_db.cc.o
[ 89%] Building CXX object CMakeFiles/rocksdb.dir/utilities/table_properties_collectors/compact_on_deletion_collector.cc.o
[ 93%] Building CXX object CMakeFiles/rocksdb.dir/utilities/transactions/optimistic_transaction_impl.cc.o
[ 93%] Building CXX object CMakeFiles/rocksdb.dir/utilities/transactions/optimistic_transaction_db_impl.cc.o
[ 93%] Building CXX object CMakeFiles/rocksdb.dir/utilities/transactions/transaction_base.cc.o
[ 93%] Building CXX object CMakeFiles/rocksdb.dir/utilities/transactions/transaction_impl.cc.o
[ 93%] Building CXX object CMakeFiles/rocksdb.dir/utilities/transactions/transaction_db_impl.cc.o
[ 93%] Building CXX object CMakeFiles/rocksdb.dir/utilities/transactions/transaction_db_mutex_impl.cc.o
[ 93%] Building CXX object CMakeFiles/rocksdb.dir/utilities/transactions/transaction_lock_mgr.cc.o
[ 96%] Building CXX object CMakeFiles/rocksdb.dir/utilities/transactions/transaction_util.cc.o
[ 96%] Building CXX object CMakeFiles/rocksdb.dir/utilities/ttl/db_ttl_impl.cc.o
[ 96%] Building CXX object CMakeFiles/rocksdb.dir/utilities/write_batch_with_index/write_batch_with_index.cc.o
[ 96%] Building CXX object CMakeFiles/rocksdb.dir/utilities/write_batch_with_index/write_batch_with_index_internal.cc.o
[ 96%] Building CXX object CMakeFiles/rocksdb.dir/utilities/col_buf_encoder.cc.o
[ 96%] Building CXX object CMakeFiles/rocksdb.dir/utilities/col_buf_decoder.cc.o
[100%] Building CXX object CMakeFiles/rocksdb.dir/utilities/column_aware_encoding_util.cc.o
[100%] Building CXX object CMakeFiles/rocksdb.dir/port/port_posix.cc.o
[100%] Building CXX object CMakeFiles/rocksdb.dir/util/env_posix.cc.o
[100%] Building CXX object CMakeFiles/rocksdb.dir/util/io_posix.cc.o
[100%] Linking CXX static library librocksdb.a
[100%] Built target rocksdb
GEN pkg/cli/zcgo_flags.go
GEN pkg/server/status/zcgo_flags.go
GEN pkg/storage/engine/zcgo_flags.go
GEN pkg/ccl/storageccl/engineccl/zcgo_flags.go
GEN pkg/cli/zcgo_flags_x86_64_pc_linux_gnu.go
GEN pkg/server/status/zcgo_flags_x86_64_pc_linux_gnu.go
GEN pkg/storage/engine/zcgo_flags_x86_64_pc_linux_gnu.go
GEN pkg/ccl/storageccl/engineccl/zcgo_flags_x86_64_pc_linux_gnu.go
go list -tags ' make x86_64_pc_linux_gnu' -f 'go test -v -race -tags '\'' make x86_64_pc_linux_gnu'\'' -ldflags '\'' -X github.com/cockroachdb/cockroach/pkg/build.typ=development'\'' -i -c {{.ImportPath}} -o '\''{{.Dir}}'\''/stress.test && (cd '\''{{.Dir}}'\'' && if [ -f stress.test ]; then COCKROACH_STRESS=true stress -maxtime 15m -maxfails 1 -stderr ./stress.test -test.run '\''.'\''  -test.timeout 30m -test.v; fi)' github.com/cockroachdb/cockroach/pkg/storage | /bin/bash
container/heap
database/sql/driver
github.com/cockroachdb/cockroach/vendor/github.com/pkg/errors
github.com/cockroachdb/cockroach/vendor/github.com/gogo/protobuf/proto
github.com/cockroachdb/cockroach/vendor/github.com/dustin/go-humanize
encoding/csv
github.com/cockroachdb/cockroach/pkg/util/syncutil
os/user
github.com/cockroachdb/cockroach/pkg/util/fileutil
github.com/cockroachdb/cockroach/pkg/util/caller
hash/adler32
compress/zlib
github.com/cockroachdb/cockroach/vendor/github.com/certifi/gocertifi
github.com/cockroachdb/cockroach/vendor/github.com/spf13/pflag
github.com/cockroachdb/cockroach/vendor/github.com/opentracing/opentracing-go/log
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/apd
database/sql
github.com/cockroachdb/cockroach/vendor/github.com/petermattis/goid
github.com/cockroachdb/cockroach/vendor/github.com/getsentry/raven-go
github.com/cockroachdb/cockroach/vendor/github.com/opentracing/opentracing-go
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/internal/timeseries
html
github.com/cockroachdb/cockroach/vendor/golang.org/x/sys/unix
html/template
github.com/cockroachdb/cockroach/vendor/github.com/satori/go.uuid
github.com/cockroachdb/cockroach/pkg/util/duration
github.com/cockroachdb/cockroach/pkg/util/uuid
github.com/cockroachdb/cockroach/pkg/util/humanizeutil
github.com/cockroachdb/cockroach/pkg/util/encoding
github.com/cockroachdb/cockroach/vendor/github.com/biogo/store/llrb
github.com/cockroachdb/cockroach/vendor/github.com/gogo/protobuf/sortkeys
github.com/cockroachdb/cockroach/pkg/settings
github.com/cockroachdb/cockroach/pkg/util/log/logflags
github.com/cockroachdb/cockroach/vendor/github.com/golang/protobuf/proto
github.com/cockroachdb/cockroach/pkg/util/interval
github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-gateway/utilities
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/hpack
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/trace
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/idna
github.com/cockroachdb/cockroach/pkg/util/envutil
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/lex/httplex
github.com/cockroachdb/cockroach/pkg/util/timeutil
github.com/cockroachdb/cockroach/pkg/util/randutil
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/codes
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/grpclog
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/keepalive
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/metadata
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/naming
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stats
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/tap
encoding/gob
github.com/cockroachdb/cockroach/vendor/github.com/lightstep/lightstep-tracer-go/thrift_0_9_2/lib/go/thrift
github.com/cockroachdb/cockroach/vendor/github.com/opentracing/opentracing-go/ext
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/peer
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2
encoding/xml
github.com/cockroachdb/cockroach/vendor/golang.org/x/crypto/blowfish
github.com/cockroachdb/cockroach/vendor/golang.org/x/crypto/ssh/terminal
github.com/cockroachdb/cockroach/pkg/build
github.com/cockroachdb/cockroach/pkg/util
github.com/cockroachdb/cockroach/vendor/github.com/gogo/protobuf/types
github.com/cockroachdb/cockroach/vendor/github.com/opentracing/basictracer-go/wire
github.com/cockroachdb/cockroach/vendor/github.com/opentracing/basictracer-go
github.com/cockroachdb/cockroach/vendor/golang.org/x/crypto/bcrypt
os/signal
github.com/cockroachdb/cockroach/vendor/github.com/rlmcpherson/s3gof3r
github.com/cockroachdb/cockroach/pkg/util/retry
github.com/cockroachdb/cockroach/pkg/util/log
github.com/cockroachdb/cockroach/vendor/gopkg.in/yaml.v2
github.com/cockroachdb/cockroach/vendor/github.com/VividCortex/ewma
github.com/cockroachdb/cockroach/vendor/github.com/cenk/backoff
github.com/cockroachdb/cockroach/vendor/github.com/codahale/hdrhistogram
github.com/cockroachdb/cockroach/vendor/github.com/golang/protobuf/jsonpb
github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-gateway/runtime/internal
github.com/cockroachdb/cockroach/vendor/github.com/golang/protobuf/ptypes/timestamp
github.com/cockroachdb/cockroach/vendor/github.com/prometheus/client_model/go
github.com/cockroachdb/cockroach/vendor/github.com/gogo/protobuf/jsonpb
github.com/cockroachdb/cockroach/vendor/github.com/matttproud/golang_protobuf_extensions/pbutil
github.com/cockroachdb/cockroach/vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg
github.com/cockroachdb/cockroach/vendor/github.com/prometheus/common/model
log/syslog
github.com/cockroachdb/cockroach/vendor/github.com/facebookgo/clock
github.com/cockroachdb/cockroach/vendor/github.com/rcrowley/go-metrics
github.com/cockroachdb/cockroach/vendor/github.com/golang/snappy
github.com/cockroachdb/cockroach/pkg/util/httputil
github.com/cockroachdb/cockroach/pkg/util/hlc
github.com/cockroachdb/cockroach/vendor/github.com/montanaflynn/stats
container/ring
github.com/cockroachdb/cockroach/vendor/github.com/lightstep/lightstep-tracer-go/lightstep_thrift
github.com/cockroachdb/cockroach/vendor/github.com/rubyist/circuitbreaker
github.com/cockroachdb/cockroach/vendor/github.com/prometheus/common/expfmt
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport
github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux
github.com/cockroachdb/cockroach/pkg/util/cache
github.com/cockroachdb/cockroach/pkg/util/shuffle
github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/singleflight
github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror
github.com/cockroachdb/cockroach/pkg/sql/privilege
github.com/cockroachdb/cockroach/vendor/github.com/knz/strtime
github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/oid
go/constant
github.com/cockroachdb/cockroach/vendor/golang.org/x/text/internal/tag
github.com/cockroachdb/cockroach/vendor/golang.org/x/text/transform
github.com/cockroachdb/cockroach/vendor/golang.org/x/text/language
github.com/cockroachdb/cockroach/pkg/util/metric
github.com/cockroachdb/cockroach/vendor/golang.org/x/text/unicode/norm
hash/fnv
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc
github.com/cockroachdb/cockroach/vendor/github.com/lightstep/lightstep-tracer-go/thrift_rpc
github.com/cockroachdb/cockroach/pkg/util/bufalloc
github.com/cockroachdb/cockroach/vendor/github.com/elastic/gosigar
github.com/cockroachdb/cockroach/pkg/sql/mon
github.com/cockroachdb/cockroach/vendor/github.com/coreos/etcd/raft/raftpb
github.com/cockroachdb/cockroach/vendor/github.com/google/btree
github.com/cockroachdb/cockroach/vendor/github.com/kr/text
github.com/cockroachdb/cockroach/vendor/golang.org/x/time/rate
github.com/cockroachdb/cockroach/vendor/github.com/kr/pretty
github.com/cockroachdb/cockroach/vendor/github.com/lib/pq
github.com/cockroachdb/cockroach/pkg/ui
github.com/cockroachdb/cockroach/pkg/util/sdnotify
github.com/cockroachdb/cockroach/vendor/github.com/coreos/etcd/raft
github.com/cockroachdb/cockroach/vendor/golang.org/x/text/internal/colltab
github.com/cockroachdb/cockroach/vendor/github.com/elazarl/go-bindata-assetfs
github.com/cockroachdb/cockroach/vendor/github.com/mitchellh/reflectwalk
expvar
net/http/pprof
github.com/cockroachdb/cockroach/pkg/util/leaktest
github.com/cockroachdb/cockroach/vendor/github.com/mattn/go-runewidth
github.com/cockroachdb/cockroach/vendor/github.com/rcrowley/go-metrics/exp
github.com/cockroachdb/cockroach/vendor/golang.org/x/text/collate
github.com/cockroachdb/cockroach/vendor/github.com/olekukonko/tablewriter
github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-gateway/runtime
github.com/cockroachdb/cockroach/vendor/github.com/lightstep/lightstep-tracer-go/collectorpb
github.com/cockroachdb/cockroach/vendor/github.com/lightstep/lightstep-tracer-go
github.com/cockroachdb/cockroach/pkg/util/protoutil
github.com/cockroachdb/cockroach/pkg/util/tracing
github.com/cockroachdb/cockroach/pkg/roachpb
github.com/cockroachdb/cockroach/pkg/util/stop
github.com/cockroachdb/cockroach/pkg/keys
github.com/cockroachdb/cockroach/pkg/sql/parser
github.com/cockroachdb/cockroach/pkg/storage/storagebase
github.com/cockroachdb/cockroach/pkg/ts/tspb
github.com/cockroachdb/cockroach/pkg/security
github.com/cockroachdb/cockroach/pkg/util/netutil
github.com/cockroachdb/cockroach/pkg/util/grpcutil
github.com/cockroachdb/cockroach/pkg/config
github.com/cockroachdb/cockroach/pkg/internal/client
github.com/cockroachdb/cockroach/pkg/base
github.com/cockroachdb/cockroach/pkg/security/securitytest
github.com/cockroachdb/cockroach/pkg/gossip/resolver
github.com/cockroachdb/cockroach/pkg/rpc
github.com/cockroachdb/cockroach/pkg/testutils
github.com/cockroachdb/cockroach/pkg/storage/engine
github.com/cockroachdb/cockroach/pkg/sql/sqlutil
github.com/cockroachdb/cockroach/pkg/server/status
github.com/cockroachdb/cockroach/pkg/gossip
github.com/cockroachdb/cockroach/pkg/kv
github.com/cockroachdb/cockroach/pkg/testutils/gossiputil
github.com/cockroachdb/cockroach/pkg/sql/sqlbase
github.com/cockroachdb/cockroach/pkg/testutils/sqlutils
github.com/cockroachdb/cockroach/pkg/testutils/serverutils
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun
github.com/cockroachdb/cockroach/pkg/storage
github.com/cockroachdb/cockroach/pkg/sql/distsqlplan
github.com/cockroachdb/cockroach/pkg/sql
github.com/cockroachdb/cockroach/pkg/server/serverpb
github.com/cockroachdb/cockroach/pkg/ts
github.com/cockroachdb/cockroach/pkg/migrations
github.com/cockroachdb/cockroach/pkg/sql/pgwire
github.com/cockroachdb/cockroach/pkg/server
github.com/cockroachdb/cockroach/pkg/testutils/testcluster
container/heap
github.com/cockroachdb/cockroach/vendor/github.com/biogo/store/llrb
github.com/cockroachdb/cockroach/vendor/github.com/gogo/protobuf/proto
github.com/cockroachdb/cockroach/vendor/github.com/dustin/go-humanize
github.com/cockroachdb/cockroach/vendor/github.com/pkg/errors
encoding/csv
github.com/cockroachdb/cockroach/pkg/util/syncutil
os/user
github.com/cockroachdb/cockroach/pkg/util/fileutil
github.com/cockroachdb/cockroach/pkg/util/caller
hash/adler32
github.com/cockroachdb/cockroach/vendor/github.com/spf13/pflag
compress/zlib
github.com/cockroachdb/cockroach/vendor/github.com/certifi/gocertifi
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/apd
github.com/cockroachdb/cockroach/vendor/github.com/opentracing/opentracing-go/log
github.com/cockroachdb/cockroach/vendor/github.com/petermattis/goid
github.com/cockroachdb/cockroach/vendor/github.com/getsentry/raven-go
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/internal/timeseries
html
github.com/cockroachdb/cockroach/vendor/github.com/opentracing/opentracing-go
html/template
github.com/cockroachdb/cockroach/vendor/golang.org/x/sys/unix
database/sql/driver
github.com/cockroachdb/cockroach/vendor/github.com/satori/go.uuid
github.com/cockroachdb/cockroach/pkg/util/duration
github.com/cockroachdb/cockroach/pkg/util/uuid
github.com/cockroachdb/cockroach/pkg/util/interval
github.com/cockroachdb/cockroach/pkg/util/humanizeutil
github.com/cockroachdb/cockroach/vendor/github.com/gogo/protobuf/sortkeys
github.com/cockroachdb/cockroach/pkg/util/encoding
github.com/cockroachdb/cockroach/pkg/settings
github.com/cockroachdb/cockroach/pkg/util/log/logflags
github.com/cockroachdb/cockroach/vendor/github.com/golang/protobuf/proto
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/trace
github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-gateway/utilities
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/hpack
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/idna
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/codes
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/lex/httplex
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/credentials
github.com/cockroachdb/cockroach/pkg/util/envutil
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/grpclog
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/keepalive
github.com/cockroachdb/cockroach/pkg/util/timeutil
github.com/cockroachdb/cockroach/pkg/util/randutil
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/metadata
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/naming
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/peer
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stats
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/tap
encoding/gob
github.com/cockroachdb/cockroach/vendor/github.com/lightstep/lightstep-tracer-go/thrift_0_9_2/lib/go/thrift
github.com/cockroachdb/cockroach/vendor/github.com/opentracing/opentracing-go/ext
encoding/xml
github.com/cockroachdb/cockroach/vendor/golang.org/x/crypto/blowfish
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2
github.com/cockroachdb/cockroach/vendor/golang.org/x/crypto/ssh/terminal
github.com/cockroachdb/cockroach/vendor/golang.org/x/crypto/bcrypt
github.com/cockroachdb/cockroach/pkg/build
github.com/cockroachdb/cockroach/pkg/util
github.com/cockroachdb/cockroach/vendor/github.com/gogo/protobuf/types
github.com/cockroachdb/cockroach/vendor/github.com/opentracing/basictracer-go/wire
github.com/cockroachdb/cockroach/vendor/github.com/opentracing/basictracer-go
github.com/cockroachdb/cockroach/vendor/github.com/rlmcpherson/s3gof3r
os/signal
github.com/cockroachdb/cockroach/pkg/util/log
github.com/cockroachdb/cockroach/vendor/github.com/golang/protobuf/jsonpb
github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-gateway/runtime/internal
github.com/cockroachdb/cockroach/vendor/github.com/golang/protobuf/ptypes/timestamp
github.com/cockroachdb/cockroach/pkg/util/retry
github.com/cockroachdb/cockroach/vendor/gopkg.in/yaml.v2
github.com/cockroachdb/cockroach/vendor/github.com/VividCortex/ewma
github.com/cockroachdb/cockroach/vendor/github.com/cenk/backoff
github.com/cockroachdb/cockroach/vendor/github.com/codahale/hdrhistogram
github.com/cockroachdb/cockroach/vendor/github.com/prometheus/client_model/go
github.com/cockroachdb/cockroach/vendor/github.com/matttproud/golang_protobuf_extensions/pbutil
github.com/cockroachdb/cockroach/vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg
github.com/cockroachdb/cockroach/vendor/github.com/prometheus/common/model
log/syslog
github.com/cockroachdb/cockroach/vendor/github.com/gogo/protobuf/jsonpb
github.com/cockroachdb/cockroach/vendor/github.com/facebookgo/clock
github.com/cockroachdb/cockroach/vendor/github.com/golang/snappy
github.com/cockroachdb/cockroach/vendor/github.com/rcrowley/go-metrics
github.com/cockroachdb/cockroach/vendor/github.com/montanaflynn/stats
github.com/cockroachdb/cockroach/vendor/github.com/prometheus/common/expfmt
github.com/cockroachdb/cockroach/pkg/util/httputil
github.com/cockroachdb/cockroach/pkg/util/hlc
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux
github.com/cockroachdb/cockroach/vendor/github.com/lightstep/lightstep-tracer-go/lightstep_thrift
github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb
container/ring
github.com/cockroachdb/cockroach/vendor/github.com/rubyist/circuitbreaker
github.com/cockroachdb/cockroach/pkg/util/bufalloc
github.com/cockroachdb/cockroach/vendor/github.com/elastic/gosigar
github.com/cockroachdb/cockroach/pkg/util/cache
github.com/cockroachdb/cockroach/pkg/util/shuffle
github.com/cockroachdb/cockroach/vendor/github.com/coreos/etcd/raft/raftpb
github.com/cockroachdb/cockroach/vendor/github.com/google/btree
github.com/cockroachdb/cockroach/vendor/github.com/kr/text
github.com/cockroachdb/cockroach/vendor/github.com/kr/pretty
github.com/cockroachdb/cockroach/pkg/util/metric
github.com/cockroachdb/cockroach/vendor/golang.org/x/time/rate
github.com/cockroachdb/cockroach/vendor/golang.org/x/sync/singleflight
github.com/cockroachdb/cockroach/pkg/util/leaktest
github.com/cockroachdb/cockroach/vendor/github.com/mattn/go-runewidth
database/sql
github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror
github.com/cockroachdb/cockroach/pkg/sql/privilege
github.com/cockroachdb/cockroach/vendor/github.com/coreos/etcd/raft
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc
github.com/cockroachdb/cockroach/vendor/github.com/knz/strtime
github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/oid
github.com/cockroachdb/cockroach/vendor/github.com/olekukonko/tablewriter
go/constant
github.com/cockroachdb/cockroach/vendor/github.com/lightstep/lightstep-tracer-go/thrift_rpc
github.com/cockroachdb/cockroach/pkg/sql/mon
github.com/cockroachdb/cockroach/vendor/golang.org/x/text/internal/tag
github.com/cockroachdb/cockroach/vendor/golang.org/x/text/language
github.com/cockroachdb/cockroach/vendor/golang.org/x/text/transform
github.com/cockroachdb/cockroach/vendor/golang.org/x/text/unicode/norm
hash/fnv
github.com/cockroachdb/cockroach/vendor/github.com/lib/pq
github.com/cockroachdb/cockroach/pkg/ui
github.com/cockroachdb/cockroach/pkg/util/sdnotify
github.com/cockroachdb/cockroach/vendor/github.com/elazarl/go-bindata-assetfs
github.com/cockroachdb/cockroach/vendor/github.com/mitchellh/reflectwalk
expvar
net/http/pprof
github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-gateway/runtime
github.com/cockroachdb/cockroach/vendor/github.com/lightstep/lightstep-tracer-go/collectorpb
github.com/cockroachdb/cockroach/vendor/github.com/rcrowley/go-metrics/exp
github.com/cockroachdb/cockroach/vendor/golang.org/x/text/internal/colltab
github.com/cockroachdb/cockroach/vendor/github.com/lightstep/lightstep-tracer-go
github.com/cockroachdb/cockroach/vendor/golang.org/x/text/collate
github.com/cockroachdb/cockroach/pkg/util/protoutil
github.com/cockroachdb/cockroach/pkg/util/tracing
github.com/cockroachdb/cockroach/pkg/roachpb
github.com/cockroachdb/cockroach/pkg/util/stop
github.com/cockroachdb/cockroach/pkg/keys
github.com/cockroachdb/cockroach/pkg/storage/storagebase
github.com/cockroachdb/cockroach/pkg/sql/parser
github.com/cockroachdb/cockroach/pkg/ts/tspb
github.com/cockroachdb/cockroach/pkg/util/netutil
github.com/cockroachdb/cockroach/pkg/security
github.com/cockroachdb/cockroach/pkg/config
github.com/cockroachdb/cockroach/pkg/internal/client
github.com/cockroachdb/cockroach/pkg/util/grpcutil
github.com/cockroachdb/cockroach/pkg/base
github.com/cockroachdb/cockroach/pkg/security/securitytest
github.com/cockroachdb/cockroach/pkg/storage/engine
github.com/cockroachdb/cockroach/pkg/gossip/resolver
github.com/cockroachdb/cockroach/pkg/rpc
github.com/cockroachdb/cockroach/pkg/testutils
github.com/cockroachdb/cockroach/pkg/sql/sqlutil
github.com/cockroachdb/cockroach/pkg/server/status
github.com/cockroachdb/cockroach/pkg/gossip
github.com/cockroachdb/cockroach/pkg/kv
github.com/cockroachdb/cockroach/pkg/testutils/gossiputil
github.com/cockroachdb/cockroach/pkg/sql/sqlbase
github.com/cockroachdb/cockroach/pkg/testutils/sqlutils
github.com/cockroachdb/cockroach/pkg/testutils/serverutils
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun
github.com/cockroachdb/cockroach/pkg/storage
github.com/cockroachdb/cockroach/pkg/storage
github.com/cockroachdb/cockroach/pkg/sql/distsqlplan
github.com/cockroachdb/cockroach/pkg/sql
github.com/cockroachdb/cockroach/pkg/server/serverpb
github.com/cockroachdb/cockroach/pkg/ts
github.com/cockroachdb/cockroach/pkg/migrations
github.com/cockroachdb/cockroach/pkg/sql/pgwire
github.com/cockroachdb/cockroach/pkg/server
github.com/cockroachdb/cockroach/pkg/testutils/testcluster
github.com/cockroachdb/cockroach/pkg/storage_test
testmain
0 runs so far, 0 failures, over 5s
0 runs so far, 0 failures, over 10s
0 runs so far, 0 failures, over 15s
0 runs so far, 0 failures, over 20s
0 runs so far, 0 failures, over 25s
0 runs so far, 0 failures, over 30s
0 runs so far, 0 failures, over 35s
0 runs so far, 0 failures, over 40s
0 runs so far, 0 failures, over 45s
0 runs so far, 0 failures, over 50s
0 runs so far, 0 failures, over 55s
0 runs so far, 0 failures, over 1m0s
0 runs so far, 0 failures, over 1m5s
0 runs so far, 0 failures, over 1m10s
0 runs so far, 0 failures, over 1m15s
0 runs so far, 0 failures, over 1m20s
0 runs so far, 0 failures, over 1m25s
0 runs so far, 0 failures, over 1m30s
0 runs so far, 0 failures, over 1m35s
0 runs so far, 0 failures, over 1m40s
0 runs so far, 0 failures, over 1m45s
0 runs so far, 0 failures, over 1m50s
0 runs so far, 0 failures, over 1m55s
0 runs so far, 0 failures, over 2m0s
0 runs so far, 0 failures, over 2m5s
0 runs so far, 0 failures, over 2m10s
0 runs so far, 0 failures, over 2m15s
0 runs so far, 0 failures, over 2m20s
0 runs so far, 0 failures, over 2m25s
0 runs so far, 0 failures, over 2m30s
0 runs so far, 0 failures, over 2m35s
0 runs so far, 0 failures, over 2m40s
0 runs so far, 0 failures, over 2m45s
0 runs so far, 0 failures, over 2m50s
0 runs so far, 0 failures, over 2m55s
0 runs so far, 0 failures, over 3m0s
0 runs so far, 0 failures, over 3m5s
1 runs so far, 0 failures, over 3m10s
4 runs so far, 0 failures, over 3m15s
5 runs so far, 0 failures, over 3m20s
7 runs so far, 0 failures, over 3m25s
8 runs so far, 0 failures, over 3m30s
8 runs so far, 0 failures, over 3m35s
8 runs so far, 0 failures, over 3m40s
8 runs so far, 0 failures, over 3m45s
8 runs so far, 0 failures, over 3m50s
8 runs so far, 0 failures, over 3m55s
8 runs so far, 0 failures, over 4m0s
8 runs so far, 0 failures, over 4m5s
8 runs so far, 0 failures, over 4m10s
8 runs so far, 0 failures, over 4m15s
8 runs so far, 0 failures, over 4m20s
8 runs so far, 0 failures, over 4m25s
8 runs so far, 0 failures, over 4m30s
8 runs so far, 0 failures, over 4m35s
8 runs so far, 0 failures, over 4m40s
8 runs so far, 0 failures, over 4m45s
8 runs so far, 0 failures, over 4m50s
8 runs so far, 0 failures, over 4m55s
8 runs so far, 0 failures, over 5m0s
8 runs so far, 0 failures, over 5m5s
8 runs so far, 0 failures, over 5m10s
8 runs so far, 0 failures, over 5m15s
8 runs so far, 0 failures, over 5m20s
8 runs so far, 0 failures, over 5m25s
8 runs so far, 0 failures, over 5m30s
8 runs so far, 0 failures, over 5m35s
8 runs so far, 0 failures, over 5m40s
8 runs so far, 0 failures, over 5m45s
8 runs so far, 0 failures, over 5m50s
8 runs so far, 0 failures, over 5m55s
8 runs so far, 0 failures, over 6m0s
9 runs so far, 0 failures, over 6m5s
9 runs so far, 0 failures, over 6m10s
9 runs so far, 0 failures, over 6m15s
11 runs so far, 0 failures, over 6m20s
14 runs so far, 0 failures, over 6m25s
15 runs so far, 0 failures, over 6m30s
16 runs so far, 0 failures, over 6m35s
16 runs so far, 0 failures, over 6m40s
16 runs so far, 0 failures, over 6m45s
16 runs so far, 0 failures, over 6m50s
16 runs so far, 0 failures, over 6m55s
16 runs so far, 0 failures, over 7m0s
16 runs so far, 0 failures, over 7m5s
16 runs so far, 0 failures, over 7m10s
16 runs so far, 0 failures, over 7m15s
16 runs so far, 0 failures, over 7m20s
16 runs so far, 0 failures, over 7m25s
16 runs so far, 0 failures, over 7m30s
16 runs so far, 0 failures, over 7m35s
16 runs so far, 0 failures, over 7m40s
16 runs so far, 0 failures, over 7m45s
16 runs so far, 0 failures, over 7m50s
16 runs so far, 0 failures, over 7m55s
16 runs so far, 0 failures, over 8m0s
16 runs so far, 0 failures, over 8m5s
16 runs so far, 0 failures, over 8m10s
16 runs so far, 0 failures, over 8m15s
16 runs so far, 0 failures, over 8m20s
16 runs so far, 0 failures, over 8m25s
16 runs so far, 0 failures, over 8m30s
16 runs so far, 0 failures, over 8m35s
16 runs so far, 0 failures, over 8m40s
16 runs so far, 0 failures, over 8m45s
16 runs so far, 0 failures, over 8m50s
17 runs so far, 0 failures, over 8m55s
17 runs so far, 0 failures, over 9m0s
17 runs so far, 0 failures, over 9m5s
17 runs so far, 0 failures, over 9m10s
17 runs so far, 0 failures, over 9m15s
18 runs so far, 0 failures, over 9m20s
19 runs so far, 0 failures, over 9m25s
23 runs so far, 0 failures, over 9m30s
24 runs so far, 0 failures, over 9m35s
24 runs so far, 0 failures, over 9m40s
24 runs so far, 0 failures, over 9m45s
24 runs so far, 0 failures, over 9m50s
24 runs so far, 0 failures, over 9m55s
24 runs so far, 0 failures, over 10m0s
24 runs so far, 0 failures, over 10m5s
24 runs so far, 0 failures, over 10m10s
24 runs so far, 0 failures, over 10m15s
24 runs so far, 0 failures, over 10m20s
24 runs so far, 0 failures, over 10m25s
24 runs so far, 0 failures, over 10m30s
24 runs so far, 0 failures, over 10m35s
24 runs so far, 0 failures, over 10m40s
24 runs so far, 0 failures, over 10m45s
24 runs so far, 0 failures, over 10m50s
24 runs so far, 0 failures, over 10m55s
24 runs so far, 0 failures, over 11m0s
24 runs so far, 0 failures, over 11m5s
24 runs so far, 0 failures, over 11m10s
24 runs so far, 0 failures, over 11m15s
24 runs so far, 0 failures, over 11m20s
24 runs so far, 0 failures, over 11m25s
24 runs so far, 0 failures, over 11m30s

I170517 07:31:45.256184 1 rand.go:76  Random seed: -8861104885856887224
=== RUN   TestAbortCachePutGetClearData
--- PASS: TestAbortCachePutGetClearData (0.01s)
=== RUN   TestAbortCacheEmptyParams
--- PASS: TestAbortCacheEmptyParams (0.01s)
=== RUN   TestAbortCacheCopyInto
--- PASS: TestAbortCacheCopyInto (0.01s)
=== RUN   TestAbortCacheCopyFrom
--- PASS: TestAbortCacheCopyFrom (0.02s)
=== RUN   TestUpdateRangeAddressing
--- PASS: TestUpdateRangeAddressing (0.14s)
=== RUN   TestUpdateRangeAddressingSplitMeta1
--- PASS: TestUpdateRangeAddressingSplitMeta1 (0.00s)
=== RUN   TestAllocatorSimpleRetrieval
--- PASS: TestAllocatorSimpleRetrieval (0.01s)
=== RUN   TestAllocatorCorruptReplica
I170517 07:31:45.475988 170 util/stop/stopper.go:505  quiescing; tasks left:
3      gossip/infostore.go:301
I170517 07:31:45.476157 170 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestAllocatorCorruptReplica (0.01s)
=== RUN   TestAllocatorNoAvailableDisks
--- PASS: TestAllocatorNoAvailableDisks (0.01s)
=== RUN   TestAllocatorTwoDatacenters
--- PASS: TestAllocatorTwoDatacenters (0.01s)
=== RUN   TestAllocatorExistingReplica
I170517 07:31:45.506490 189 util/stop/stopper.go:505  quiescing; tasks left:
4      gossip/infostore.go:301
--- PASS: TestAllocatorExistingReplica (0.01s)
=== RUN   TestAllocatorRelaxConstraints
=== RUN   TestAllocatorRelaxConstraints/positive_constraints_(matching_store_1)
=== RUN   TestAllocatorRelaxConstraints/positive_constraints_(matching_store_2)
=== RUN   TestAllocatorRelaxConstraints/positive_constraints_(matching_store_1)_with_existing_replica_(store_1)
=== RUN   TestAllocatorRelaxConstraints/positive_constraints_(matching_store_1)_with_two_existing_replicas
=== RUN   TestAllocatorRelaxConstraints/positive_constraints_(matching_store_2)_with_two_existing_replicas
=== RUN   TestAllocatorRelaxConstraints/required_constraints_(matching_store_1)_with_existing_replica_(store_1)
=== RUN   TestAllocatorRelaxConstraints/required_constraints_(matching_store_2)_with_exiting_replica_(store_2)
=== RUN   TestAllocatorRelaxConstraints/positive_constraints_(matching_store_2)_with_existing_replica_(store_1)
=== RUN   TestAllocatorRelaxConstraints/positive_constraints_(matching_store_2)_with_existing_replica_(store_2)
=== RUN   TestAllocatorRelaxConstraints/positive_constraints_(half_matching_store_2)
=== RUN   TestAllocatorRelaxConstraints/positive_constraints_(half_matching_store_2)_with_existing_replica_(store_2)
=== RUN   TestAllocatorRelaxConstraints/required_constraints_(half_matching_store_2)_with_existing_replica_(store_2)
=== RUN   TestAllocatorRelaxConstraints/positive_constraints_(half_matching_store_2)_with_two_existing_replica
=== RUN   TestAllocatorRelaxConstraints/positive_constraints_(2/3_matching_store_2)
=== RUN   TestAllocatorRelaxConstraints/positive_constraints_(1/3_matching_store_2)
--- PASS: TestAllocatorRelaxConstraints (0.02s)
    --- PASS: TestAllocatorRelaxConstraints/positive_constraints_(matching_store_1) (0.00s)
    --- PASS: TestAllocatorRelaxConstraints/positive_constraints_(matching_store_2) (0.00s)
    --- PASS: TestAllocatorRelaxConstraints/positive_constraints_(matching_store_1)_with_existing_replica_(store_1) (0.00s)
    --- PASS: TestAllocatorRelaxConstraints/positive_constraints_(matching_store_1)_with_two_existing_replicas (0.00s)
    --- PASS: TestAllocatorRelaxConstraints/positive_constraints_(matching_store_2)_with_two_existing_replicas (0.00s)
    --- PASS: TestAllocatorRelaxConstraints/required_constraints_(matching_store_1)_with_existing_replica_(store_1) (0.00s)
    --- PASS: TestAllocatorRelaxConstraints/required_constraints_(matching_store_2)_with_exiting_replica_(store_2) (0.00s)
    --- PASS: TestAllocatorRelaxConstraints/positive_constraints_(matching_store_2)_with_existing_replica_(store_1) (0.00s)
    --- PASS: TestAllocatorRelaxConstraints/positive_constraints_(matching_store_2)_with_existing_replica_(store_2) (0.00s)
    --- PASS: TestAllocatorRelaxConstraints/positive_constraints_(half_matching_store_2) (0.00s)
    --- PASS: TestAllocatorRelaxConstraints/positive_constraints_(half_matching_store_2)_with_existing_replica_(store_2) (0.00s)
    --- PASS: TestAllocatorRelaxConstraints/required_constraints_(half_matching_store_2)_with_existing_replica_(store_2) (0.00s)
    --- PASS: TestAllocatorRelaxConstraints/positive_constraints_(half_matching_store_2)_with_two_existing_replica (0.00s)
    --- PASS: TestAllocatorRelaxConstraints/positive_constraints_(2/3_matching_store_2) (0.00s)
    --- PASS: TestAllocatorRelaxConstraints/positive_constraints_(1/3_matching_store_2) (0.00s)
=== RUN   TestAllocatorRebalance
--- PASS: TestAllocatorRebalance (0.01s)
=== RUN   TestAllocatorRebalanceDeadNodes
=== RUN   TestAllocatorRebalanceDeadNodes/#00
=== RUN   TestAllocatorRebalanceDeadNodes/#01
=== RUN   TestAllocatorRebalanceDeadNodes/#02
=== RUN   TestAllocatorRebalanceDeadNodes/#03
=== RUN   TestAllocatorRebalanceDeadNodes/#04
=== RUN   TestAllocatorRebalanceDeadNodes/#05
=== RUN   TestAllocatorRebalanceDeadNodes/#06
--- PASS: TestAllocatorRebalanceDeadNodes (0.01s)
    --- PASS: TestAllocatorRebalanceDeadNodes/#00 (0.00s)
    --- PASS: TestAllocatorRebalanceDeadNodes/#01 (0.00s)
    --- PASS: TestAllocatorRebalanceDeadNodes/#02 (0.00s)
    --- PASS: TestAllocatorRebalanceDeadNodes/#03 (0.00s)
    --- PASS: TestAllocatorRebalanceDeadNodes/#04 (0.00s)
    --- PASS: TestAllocatorRebalanceDeadNodes/#05 (0.00s)
    --- PASS: TestAllocatorRebalanceDeadNodes/#06 (0.00s)
=== RUN   TestAllocatorRebalanceThrashing
=== RUN   TestAllocatorRebalanceThrashing/balanced
=== RUN   TestAllocatorRebalanceThrashing/empty-node
I170517 07:31:45.576258 205 util/stop/stopper.go:505  quiescing; tasks left:
2      gossip/infostore.go:301
=== RUN   TestAllocatorRebalanceThrashing/within-threshold
=== RUN   TestAllocatorRebalanceThrashing/5-stores-mean-100-one-above
=== RUN   TestAllocatorRebalanceThrashing/5-stores-mean-1000-one-above
=== RUN   TestAllocatorRebalanceThrashing/5-stores-mean-10000-one-above
=== RUN   TestAllocatorRebalanceThrashing/5-stores-mean-1000-one-underused
=== RUN   TestAllocatorRebalanceThrashing/10-stores-mean-1000-one-underused
I170517 07:31:45.597162 253 util/stop/stopper.go:505  quiescing; tasks left:
3      gossip/infostore.go:301
I170517 07:31:45.597296 253 util/stop/stopper.go:505  quiescing; tasks left:
2      gossip/infostore.go:301
I170517 07:31:45.597386 253 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestAllocatorRebalanceThrashing (0.04s)
    --- PASS: TestAllocatorRebalanceThrashing/balanced (0.00s)
    --- PASS: TestAllocatorRebalanceThrashing/empty-node (0.00s)
    --- PASS: TestAllocatorRebalanceThrashing/within-threshold (0.00s)
    --- PASS: TestAllocatorRebalanceThrashing/5-stores-mean-100-one-above (0.01s)
    --- PASS: TestAllocatorRebalanceThrashing/5-stores-mean-1000-one-above (0.00s)
    --- PASS: TestAllocatorRebalanceThrashing/5-stores-mean-10000-one-above (0.00s)
    --- PASS: TestAllocatorRebalanceThrashing/5-stores-mean-1000-one-underused (0.00s)
    --- PASS: TestAllocatorRebalanceThrashing/10-stores-mean-1000-one-underused (0.00s)
=== RUN   TestAllocatorRebalanceByCount
--- PASS: TestAllocatorRebalanceByCount (0.02s)
=== RUN   TestAllocatorTransferLeaseTarget
=== RUN   TestAllocatorTransferLeaseTarget/#00
=== RUN   TestAllocatorTransferLeaseTarget/#01
=== RUN   TestAllocatorTransferLeaseTarget/#02
=== RUN   TestAllocatorTransferLeaseTarget/#03
=== RUN   TestAllocatorTransferLeaseTarget/#04
=== RUN   TestAllocatorTransferLeaseTarget/#05
=== RUN   TestAllocatorTransferLeaseTarget/#06
--- PASS: TestAllocatorTransferLeaseTarget (0.02s)
    --- PASS: TestAllocatorTransferLeaseTarget/#00 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTarget/#01 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTarget/#02 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTarget/#03 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTarget/#04 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTarget/#05 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTarget/#06 (0.00s)
=== RUN   TestAllocatorTransferLeaseTargetMultiStore
=== RUN   TestAllocatorTransferLeaseTargetMultiStore/#00
=== RUN   TestAllocatorTransferLeaseTargetMultiStore/#01
=== RUN   TestAllocatorTransferLeaseTargetMultiStore/#02
--- PASS: TestAllocatorTransferLeaseTargetMultiStore (0.03s)
    --- PASS: TestAllocatorTransferLeaseTargetMultiStore/#00 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetMultiStore/#01 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetMultiStore/#02 (0.00s)
=== RUN   TestAllocatorShouldTransferLease
=== RUN   TestAllocatorShouldTransferLease/#00
=== RUN   TestAllocatorShouldTransferLease/#01
=== RUN   TestAllocatorShouldTransferLease/#02
=== RUN   TestAllocatorShouldTransferLease/#03
=== RUN   TestAllocatorShouldTransferLease/#04
=== RUN   TestAllocatorShouldTransferLease/#05
=== RUN   TestAllocatorShouldTransferLease/#06
=== RUN   TestAllocatorShouldTransferLease/#07
=== RUN   TestAllocatorShouldTransferLease/#08
--- PASS: TestAllocatorShouldTransferLease (0.02s)
    --- PASS: TestAllocatorShouldTransferLease/#00 (0.00s)
    --- PASS: TestAllocatorShouldTransferLease/#01 (0.00s)
    --- PASS: TestAllocatorShouldTransferLease/#02 (0.00s)
    --- PASS: TestAllocatorShouldTransferLease/#03 (0.00s)
    --- PASS: TestAllocatorShouldTransferLease/#04 (0.00s)
    --- PASS: TestAllocatorShouldTransferLease/#05 (0.00s)
    --- PASS: TestAllocatorShouldTransferLease/#06 (0.00s)
    --- PASS: TestAllocatorShouldTransferLease/#07 (0.00s)
    --- PASS: TestAllocatorShouldTransferLease/#08 (0.00s)
=== RUN   TestAllocatorTransferLeaseTargetLoadBased
I170517 07:31:45.693811 357 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"1" > attrs:<> locality:<tiers:<key:"l" value:"1" > >
I170517 07:31:45.694145 357 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"2" > attrs:<> locality:<tiers:<key:"l" value:"2" > >
I170517 07:31:45.694429 357 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"3" > attrs:<> locality:<tiers:<key:"l" value:"3" > >
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#00
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#01
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#02
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#03
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#04
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#05
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#06
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#07
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#08
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#09
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#10
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#11
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#12
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#13
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#14
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#15
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#16
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#17
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#18
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#19
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#20
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#21
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#22
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#23
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#24
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#25
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#26
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#27
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#28
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#29
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#30
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#31
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#32
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#33
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#34
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#35
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#36
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#37
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#38
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#39
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#40
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#41
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#42
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#43
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#44
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#45
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#46
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#47
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#48
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#49
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#50
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#51
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#52
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#53
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#54
=== RUN   TestAllocatorTransferLeaseTargetLoadBased/#55
--- PASS: TestAllocatorTransferLeaseTargetLoadBased (0.04s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#00 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#01 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#02 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#03 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#04 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#05 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#06 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#07 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#08 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#09 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#10 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#11 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#12 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#13 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#14 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#15 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#16 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#17 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#18 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#19 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#20 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#21 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#22 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#23 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#24 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#25 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#26 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#27 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#28 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#29 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#30 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#31 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#32 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#33 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#34 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#35 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#36 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#37 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#38 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#39 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#40 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#41 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#42 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#43 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#44 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#45 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#46 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#47 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#48 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#49 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#50 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#51 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#52 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#53 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#54 (0.00s)
    --- PASS: TestAllocatorTransferLeaseTargetLoadBased/#55 (0.00s)
=== RUN   TestLoadBasedLeaseRebalanceScore
--- PASS: TestLoadBasedLeaseRebalanceScore (0.01s)
=== RUN   TestAllocatorRemoveTarget
I170517 07:31:45.749077 380 util/stop/stopper.go:505  quiescing; tasks left:
4      gossip/infostore.go:301
I170517 07:31:45.750432 380 util/stop/stopper.go:505  quiescing; tasks left:
3      gossip/infostore.go:301
--- PASS: TestAllocatorRemoveTarget (0.01s)
=== RUN   TestAllocatorComputeAction
--- PASS: TestAllocatorComputeAction (0.01s)
=== RUN   TestAllocatorComputeActionNoStorePool
--- PASS: TestAllocatorComputeActionNoStorePool (0.01s)
=== RUN   TestAllocatorError
--- PASS: TestAllocatorError (0.01s)
=== RUN   TestAllocatorThrottled
--- PASS: TestAllocatorThrottled (0.02s)
=== RUN   TestFilterBehindReplicas
=== RUN   TestFilterBehindReplicas/#00
=== RUN   TestFilterBehindReplicas/#01
=== RUN   TestFilterBehindReplicas/#02
=== RUN   TestFilterBehindReplicas/#03
=== RUN   TestFilterBehindReplicas/#04
=== RUN   TestFilterBehindReplicas/#05
=== RUN   TestFilterBehindReplicas/#06
=== RUN   TestFilterBehindReplicas/#07
=== RUN   TestFilterBehindReplicas/#08
=== RUN   TestFilterBehindReplicas/#09
=== RUN   TestFilterBehindReplicas/#10
=== RUN   TestFilterBehindReplicas/#11
=== RUN   TestFilterBehindReplicas/#12
=== RUN   TestFilterBehindReplicas/#13
=== RUN   TestFilterBehindReplicas/#14
=== RUN   TestFilterBehindReplicas/#15
=== RUN   TestFilterBehindReplicas/#16
=== RUN   TestFilterBehindReplicas/#17
=== RUN   TestFilterBehindReplicas/#18
=== RUN   TestFilterBehindReplicas/#19
=== RUN   TestFilterBehindReplicas/#20
=== RUN   TestFilterBehindReplicas/#21
--- PASS: TestFilterBehindReplicas (0.01s)
    --- PASS: TestFilterBehindReplicas/#00 (0.00s)
    --- PASS: TestFilterBehindReplicas/#01 (0.00s)
    --- PASS: TestFilterBehindReplicas/#02 (0.00s)
    --- PASS: TestFilterBehindReplicas/#03 (0.00s)
    --- PASS: TestFilterBehindReplicas/#04 (0.00s)
    --- PASS: TestFilterBehindReplicas/#05 (0.00s)
    --- PASS: TestFilterBehindReplicas/#06 (0.00s)
    --- PASS: TestFilterBehindReplicas/#07 (0.00s)
    --- PASS: TestFilterBehindReplicas/#08 (0.00s)
    --- PASS: TestFilterBehindReplicas/#09 (0.00s)
    --- PASS: TestFilterBehindReplicas/#10 (0.00s)
    --- PASS: TestFilterBehindReplicas/#11 (0.00s)
    --- PASS: TestFilterBehindReplicas/#12 (0.00s)
    --- PASS: TestFilterBehindReplicas/#13 (0.00s)
    --- PASS: TestFilterBehindReplicas/#14 (0.00s)
    --- PASS: TestFilterBehindReplicas/#15 (0.00s)
    --- PASS: TestFilterBehindReplicas/#16 (0.00s)
    --- PASS: TestFilterBehindReplicas/#17 (0.00s)
    --- PASS: TestFilterBehindReplicas/#18 (0.00s)
    --- PASS: TestFilterBehindReplicas/#19 (0.00s)
    --- PASS: TestFilterBehindReplicas/#20 (0.00s)
    --- PASS: TestFilterBehindReplicas/#21 (0.00s)
=== RUN   TestAllocatorRebalanceAway
=== RUN   TestAllocatorRebalanceAway/+datacenter=us
=== RUN   TestAllocatorRebalanceAway/-datacenter=eur
=== RUN   TestAllocatorRebalanceAway/+datacenter=eur
=== RUN   TestAllocatorRebalanceAway/-datacenter=us
=== RUN   TestAllocatorRebalanceAway/+datacenter=other
=== RUN   TestAllocatorRebalanceAway/-datacenter=other
=== RUN   TestAllocatorRebalanceAway/datacenter=other
--- PASS: TestAllocatorRebalanceAway (0.01s)
    --- PASS: TestAllocatorRebalanceAway/+datacenter=us (0.00s)
    --- PASS: TestAllocatorRebalanceAway/-datacenter=eur (0.00s)
    --- PASS: TestAllocatorRebalanceAway/+datacenter=eur (0.00s)
    --- PASS: TestAllocatorRebalanceAway/-datacenter=us (0.00s)
    --- PASS: TestAllocatorRebalanceAway/+datacenter=other (0.00s)
    --- PASS: TestAllocatorRebalanceAway/-datacenter=other (0.00s)
    --- PASS: TestAllocatorRebalanceAway/datacenter=other (0.00s)
=== RUN   TestCommandQueue
--- PASS: TestCommandQueue (0.01s)
=== RUN   TestCommandQueueWriteWaitForNonAdjacentRead
--- PASS: TestCommandQueueWriteWaitForNonAdjacentRead (0.01s)
=== RUN   TestCommandQueueNoWaitOnReadOnly
--- PASS: TestCommandQueueNoWaitOnReadOnly (0.01s)
=== RUN   TestCommandQueueMultipleExecutingCommands
--- PASS: TestCommandQueueMultipleExecutingCommands (0.02s)
=== RUN   TestCommandQueueMultiplePendingCommands
--- PASS: TestCommandQueueMultiplePendingCommands (0.02s)
=== RUN   TestCommandQueueRemove
--- PASS: TestCommandQueueRemove (0.01s)
=== RUN   TestCommandQueueExclusiveEnd
--- PASS: TestCommandQueueExclusiveEnd (0.00s)
=== RUN   TestCommandQueueSelfOverlap
--- PASS: TestCommandQueueSelfOverlap (0.01s)
=== RUN   TestCommandQueueCoveringOptimization
--- PASS: TestCommandQueueCoveringOptimization (0.02s)
=== RUN   TestCommandQueueWithoutCoveringOptimization
--- PASS: TestCommandQueueWithoutCoveringOptimization (0.01s)
=== RUN   TestCommandQueueIssue6495
--- PASS: TestCommandQueueIssue6495 (0.00s)
=== RUN   TestCommandQueueTimestamps
--- PASS: TestCommandQueueTimestamps (0.01s)
=== RUN   TestCommandQueueEnclosed
--- PASS: TestCommandQueueEnclosed (0.01s)
=== RUN   TestCommandQueueTimestampsEmpty
--- PASS: TestCommandQueueTimestampsEmpty (0.00s)
=== RUN   TestEntryCache
--- PASS: TestEntryCache (0.03s)
=== RUN   TestEntryCacheClearTo
--- PASS: TestEntryCacheClearTo (0.01s)
=== RUN   TestEntryCacheEviction
--- PASS: TestEntryCacheEviction (0.01s)
=== RUN   TestGCQueueShouldQueue
I170517 07:31:46.077795 476 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:46.105964 476 storage/gc_queue.go:148  gcScore = 0.000000, intentScore = 15.000000, ms={ContainsEstimates:false LastUpdateNanos:864000000000000 IntentAge:1296000 GCBytesAge:0 LiveBytes:0 LiveCount:0 KeyBytes:0 KeyCount:0 ValBytes:0 ValCount:0 IntentBytes:0 IntentCount:1 SysBytes:0 SysCount:0}
I170517 07:31:46.106107 476 storage/gc_queue.go:148  gcScore = 0.000000, intentScore = 35.000000, ms={ContainsEstimates:false LastUpdateNanos:864000000000000 IntentAge:6048000 GCBytesAge:0 LiveBytes:0 LiveCount:0 KeyBytes:0 KeyCount:0 ValBytes:0 ValCount:0 IntentBytes:0 IntentCount:2 SysBytes:0 SysCount:0}
I170517 07:31:46.106348 476 storage/gc_queue.go:148  gcScore = 20.000000, intentScore = 0.000000, ms={ContainsEstimates:false LastUpdateNanos:864000000000000 IntentAge:0 GCBytesAge:1811939328000 LiveBytes:0 LiveCount:0 KeyBytes:1048576 KeyCount:0 ValBytes:0 ValCount:0 IntentBytes:0 IntentCount:0 SysBytes:0 SysCount:0}
I170517 07:31:46.106481 476 storage/gc_queue.go:148  gcScore = 20.000000, intentScore = 0.000000, ms={ContainsEstimates:false LastUpdateNanos:864000000000000 IntentAge:0 GCBytesAge:1811939328000 LiveBytes:0 LiveCount:0 KeyBytes:2097152 KeyCount:0 ValBytes:0 ValCount:0 IntentBytes:0 IntentCount:0 SysBytes:0 SysCount:0}
I170517 07:31:46.106608 476 storage/gc_queue.go:148  gcScore = 50.000000, intentScore = 0.000000, ms={ContainsEstimates:false LastUpdateNanos:864000000000000 IntentAge:0 GCBytesAge:4529848320000 LiveBytes:0 LiveCount:0 KeyBytes:1048576 KeyCount:0 ValBytes:0 ValCount:0 IntentBytes:0 IntentCount:864000 SysBytes:0 SysCount:0}
I170517 07:31:46.106748 476 storage/gc_queue.go:148  gcScore = 10.000000, intentScore = 20.000000, ms={ContainsEstimates:false LastUpdateNanos:864000000000000 IntentAge:8640000 GCBytesAge:905969664000 LiveBytes:0 LiveCount:0 KeyBytes:1048576 KeyCount:0 ValBytes:0 ValCount:0 IntentBytes:0 IntentCount:5 SysBytes:0 SysCount:0}
I170517 07:31:46.106870 476 storage/gc_queue.go:148  gcScore = 864000.000000, intentScore = 0.000000, ms={ContainsEstimates:false LastUpdateNanos:0 IntentAge:0 GCBytesAge:0 LiveBytes:0 LiveCount:0 KeyBytes:90596966400 KeyCount:0 ValBytes:0 ValCount:0 IntentBytes:0 IntentCount:0 SysBytes:0 SysCount:0}
I170517 07:31:46.106988 476 storage/gc_queue.go:148  gcScore = 0.000000, intentScore = 10.000000, ms={ContainsEstimates:false LastUpdateNanos:0 IntentAge:0 GCBytesAge:0 LiveBytes:0 LiveCount:0 KeyBytes:0 KeyCount:0 ValBytes:0 ValCount:0 IntentBytes:0 IntentCount:2 SysBytes:0 SysCount:0}
--- PASS: TestGCQueueShouldQueue (0.07s)
=== RUN   TestGCQueueProcess
I170517 07:31:46.152593 585 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestGCQueueProcess (0.10s)
=== RUN   TestGCQueueTransactionTable
I170517 07:31:46.259525 484 storage/replica_command.go:177  [s1,r1/1:/M{in-ax}] test injecting error: boom
W170517 07:31:46.259806 484 storage/gc_queue.go:227  unable to resolve intents of committed txn on gc: boom
--- PASS: TestGCQueueTransactionTable (0.10s)
=== RUN   TestGCQueueIntentResolution
I170517 07:31:46.345262 669 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestGCQueueIntentResolution (0.06s)
=== RUN   TestGCQueueLastProcessedTimestamps
I170517 07:31:46.425827 742 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestGCQueueLastProcessedTimestamps (0.07s)
=== RUN   TestIDAllocator
--- PASS: TestIDAllocator (0.04s)
=== RUN   TestIDAllocatorNegativeValue
I170517 07:31:46.676641 953 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/id_alloc.go:108
--- PASS: TestIDAllocatorNegativeValue (0.18s)
=== RUN   TestNewIDAllocatorInvalidArgs
--- PASS: TestNewIDAllocatorInvalidArgs (0.00s)
=== RUN   TestAllocateErrorAndRecovery
W170517 07:31:46.716584 1056 storage/id_alloc.go:117  unable to allocate 10 ids from /Min: attempted access to empty key
I170517 07:31:46.731465 1018 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/id_alloc.go:108
W170517 07:31:46.731688 1056 storage/replica.go:2296  [s1,r1/1:/M{in-ax}] shutdown cancellation after 0.0s of attempting command Increment [/System/"range-idgen",/Min)
W170517 07:31:46.731800 1056 storage/id_alloc.go:117  unable to allocate 10 ids from /System/"range-idgen": result is ambiguous (server shutdown)
W170517 07:31:46.743257 1056 storage/id_alloc.go:109  node unavailable; try another peer
--- PASS: TestAllocateErrorAndRecovery (0.05s)
=== RUN   TestAllocateWithStopper
W170517 07:31:46.784183 1336 storage/id_alloc.go:109  node unavailable; try another peer
--- PASS: TestAllocateWithStopper (0.04s)
=== RUN   TestPushTransactionsWithNonPendingIntent
I170517 07:31:46.798810 1337 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestPushTransactionsWithNonPendingIntent (0.04s)
=== RUN   TestLeaseHistory
--- PASS: TestLeaseHistory (0.01s)
=== RUN   TestShouldPushImmediately
=== RUN   TestShouldPushImmediately/#00
=== RUN   TestShouldPushImmediately/#01
=== RUN   TestShouldPushImmediately/#02
=== RUN   TestShouldPushImmediately/#03
=== RUN   TestShouldPushImmediately/#04
=== RUN   TestShouldPushImmediately/#05
=== RUN   TestShouldPushImmediately/#06
=== RUN   TestShouldPushImmediately/#07
=== RUN   TestShouldPushImmediately/#08
=== RUN   TestShouldPushImmediately/#09
=== RUN   TestShouldPushImmediately/#10
=== RUN   TestShouldPushImmediately/#11
=== RUN   TestShouldPushImmediately/#12
=== RUN   TestShouldPushImmediately/#13
=== RUN   TestShouldPushImmediately/#14
=== RUN   TestShouldPushImmediately/#15
=== RUN   TestShouldPushImmediately/#16
=== RUN   TestShouldPushImmediately/#17
=== RUN   TestShouldPushImmediately/#18
=== RUN   TestShouldPushImmediately/#19
=== RUN   TestShouldPushImmediately/#20
=== RUN   TestShouldPushImmediately/#21
=== RUN   TestShouldPushImmediately/#22
=== RUN   TestShouldPushImmediately/#23
=== RUN   TestShouldPushImmediately/#24
=== RUN   TestShouldPushImmediately/#25
=== RUN   TestShouldPushImmediately/#26
--- PASS: TestShouldPushImmediately (0.01s)
    --- PASS: TestShouldPushImmediately/#00 (0.00s)
    --- PASS: TestShouldPushImmediately/#01 (0.00s)
    --- PASS: TestShouldPushImmediately/#02 (0.00s)
    --- PASS: TestShouldPushImmediately/#03 (0.00s)
    --- PASS: TestShouldPushImmediately/#04 (0.00s)
    --- PASS: TestShouldPushImmediately/#05 (0.00s)
    --- PASS: TestShouldPushImmediately/#06 (0.00s)
    --- PASS: TestShouldPushImmediately/#07 (0.00s)
    --- PASS: TestShouldPushImmediately/#08 (0.00s)
    --- PASS: TestShouldPushImmediately/#09 (0.00s)
    --- PASS: TestShouldPushImmediately/#10 (0.00s)
    --- PASS: TestShouldPushImmediately/#11 (0.00s)
    --- PASS: TestShouldPushImmediately/#12 (0.00s)
    --- PASS: TestShouldPushImmediately/#13 (0.00s)
    --- PASS: TestShouldPushImmediately/#14 (0.00s)
    --- PASS: TestShouldPushImmediately/#15 (0.00s)
    --- PASS: TestShouldPushImmediately/#16 (0.00s)
    --- PASS: TestShouldPushImmediately/#17 (0.00s)
    --- PASS: TestShouldPushImmediately/#18 (0.00s)
    --- PASS: TestShouldPushImmediately/#19 (0.00s)
    --- PASS: TestShouldPushImmediately/#20 (0.00s)
    --- PASS: TestShouldPushImmediately/#21 (0.00s)
    --- PASS: TestShouldPushImmediately/#22 (0.00s)
    --- PASS: TestShouldPushImmediately/#23 (0.00s)
    --- PASS: TestShouldPushImmediately/#24 (0.00s)
    --- PASS: TestShouldPushImmediately/#25 (0.00s)
    --- PASS: TestShouldPushImmediately/#26 (0.00s)
=== RUN   TestIsPushed
=== RUN   TestIsPushed/#00
=== RUN   TestIsPushed/#01
=== RUN   TestIsPushed/#02
=== RUN   TestIsPushed/#03
=== RUN   TestIsPushed/#04
=== RUN   TestIsPushed/#05
=== RUN   TestIsPushed/#06
=== RUN   TestIsPushed/#07
=== RUN   TestIsPushed/#08
--- PASS: TestIsPushed (0.01s)
    --- PASS: TestIsPushed/#00 (0.00s)
    --- PASS: TestIsPushed/#01 (0.00s)
    --- PASS: TestIsPushed/#02 (0.00s)
    --- PASS: TestIsPushed/#03 (0.00s)
    --- PASS: TestIsPushed/#04 (0.00s)
    --- PASS: TestIsPushed/#05 (0.00s)
    --- PASS: TestIsPushed/#06 (0.00s)
    --- PASS: TestIsPushed/#07 (0.00s)
    --- PASS: TestIsPushed/#08 (0.00s)
=== RUN   TestPushTxnQueueEnableDisable
I170517 07:31:46.864370 1458 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:46.869656 1458 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestPushTxnQueueEnableDisable (0.03s)
=== RUN   TestPushTxnQueueCancel
I170517 07:31:46.916569 1462 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestPushTxnQueueCancel (0.10s)
=== RUN   TestPushTxnQueueUpdateTxn
--- PASS: TestPushTxnQueueUpdateTxn (0.11s)
=== RUN   TestPushTxnQueueUpdateNotPushedTxn
--- PASS: TestPushTxnQueueUpdateNotPushedTxn (0.10s)
=== RUN   TestPushTxnQueuePusheeExpires
I170517 07:31:47.195800 1742 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestPushTxnQueuePusheeExpires (0.05s)
=== RUN   TestPushTxnQueuePusherUpdate
I170517 07:31:47.248990 1939 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestPushTxnQueuePusherUpdate (0.04s)
=== RUN   TestPushTxnQueueDependencyCycle
I170517 07:31:47.303376 2025 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestPushTxnQueueDependencyCycle (0.12s)
=== RUN   TestPushTxnQueueDependencyCycleWithPriorityInversion
I170517 07:31:47.406081 1983 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestPushTxnQueueDependencyCycleWithPriorityInversion (0.11s)
=== RUN   TestQueuePriorityQueue
--- PASS: TestQueuePriorityQueue (0.00s)
=== RUN   TestBaseQueueAddUpdateAndRemove
I170517 07:31:47.538843 2226 storage/store.go:2139  removing replica
I170517 07:31:47.539194 2226 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
--- PASS: TestBaseQueueAddUpdateAndRemove (0.04s)
=== RUN   TestBaseQueueAdd
I170517 07:31:47.580238 1901 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:47.585700 1901 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestBaseQueueAdd (0.10s)
=== RUN   TestBaseQueueProcess
I170517 07:31:47.663441 2402 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:47.668163 2402 storage/store.go:2139  removing replica
I170517 07:31:47.668433 2402 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
--- PASS: TestBaseQueueProcess (0.12s)
=== RUN   TestBaseQueueAddRemove
--- PASS: TestBaseQueueAddRemove (0.03s)
=== RUN   TestAcceptsUnsplitRanges
I170517 07:31:47.837568 2572 storage/store.go:2139  removing replica
I170517 07:31:47.837880 2572 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
--- PASS: TestAcceptsUnsplitRanges (0.05s)
=== RUN   TestBaseQueuePurgatory
I170517 07:31:47.874078 1551 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:47.879320 1551 storage/store.go:2139  removing replica
I170517 07:31:47.879611 1551 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:31:47.890141 2763 storage/queue.go:699  [s1,test] purgatory is now empty
--- PASS: TestBaseQueuePurgatory (0.05s)
=== RUN   TestBaseQueueProcessTimeout
I170517 07:31:47.917373 2121 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
E170517 07:31:47.924969 2846 storage/queue.go:634  [test,s1,r1/1:/M{in-ax}] context deadline exceeded
I170517 07:31:47.925110 2121 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/queue.go:523
--- PASS: TestBaseQueueProcessTimeout (0.04s)
=== RUN   TestBaseQueueTimeMetric
I170517 07:31:47.949638 2850 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestBaseQueueTimeMetric (0.09s)
=== RUN   TestBaseQueueShouldQueueAgain
--- PASS: TestBaseQueueShouldQueueAgain (0.00s)
=== RUN   TestBaseQueueDisable
I170517 07:31:48.056779 2577 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:48.057392 2577 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestBaseQueueDisable (0.08s)
=== RUN   TestShouldTruncate
=== RUN   TestShouldTruncate/#00
=== RUN   TestShouldTruncate/#01
=== RUN   TestShouldTruncate/#02
=== RUN   TestShouldTruncate/#03
=== RUN   TestShouldTruncate/#04
--- PASS: TestShouldTruncate (0.02s)
    --- PASS: TestShouldTruncate/#00 (0.00s)
    --- PASS: TestShouldTruncate/#01 (0.00s)
    --- PASS: TestShouldTruncate/#02 (0.00s)
    --- PASS: TestShouldTruncate/#03 (0.00s)
    --- PASS: TestShouldTruncate/#04 (0.00s)
=== RUN   TestGetQuorumIndex
--- PASS: TestGetQuorumIndex (0.01s)
=== RUN   TestComputeTruncatableIndex
--- PASS: TestComputeTruncatableIndex (0.01s)
=== RUN   TestGetTruncatableIndexes
--- PASS: TestGetTruncatableIndexes (0.19s)
=== RUN   TestProactiveRaftLogTruncate
=== RUN   TestProactiveRaftLogTruncate/#00
=== RUN   TestProactiveRaftLogTruncate/#01
--- PASS: TestProactiveRaftLogTruncate (0.37s)
    --- PASS: TestProactiveRaftLogTruncate/#00 (0.34s)
    --- PASS: TestProactiveRaftLogTruncate/#01 (0.03s)
=== RUN   TestReplicaDataIteratorEmptyRange
I170517 07:31:48.729232 3180 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaDataIteratorEmptyRange (0.08s)
=== RUN   TestReplicaDataIterator
I170517 07:31:48.796387 3269 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:48.802149 3269 storage/store.go:2139  removing replica
I170517 07:31:48.802438 3269 storage/replica.go:684  removed 6 (0+6) keys in 0ms [clear=0ms commit=0ms]
--- PASS: TestReplicaDataIterator (0.07s)
=== RUN   TestReplicaGCShouldQueue
--- PASS: TestReplicaGCShouldQueue (0.01s)
=== RUN   TestSkipLargeReplicaSnapshot
I170517 07:31:48.898631 3433 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated test snapshot e20eeb6c at index 17
--- PASS: TestSkipLargeReplicaSnapshot (0.05s)
=== RUN   TestSynthesizeHardState
--- PASS: TestSynthesizeHardState (0.07s)
=== RUN   TestReplicaStats
--- PASS: TestReplicaStats (0.00s)
=== RUN   TestReplicaStatsDecay
--- PASS: TestReplicaStatsDecay (0.01s)
=== RUN   TestReplicaStatsDecaySmoothing
--- PASS: TestReplicaStatsDecaySmoothing (0.01s)
=== RUN   TestIsOnePhaseCommit
--- PASS: TestIsOnePhaseCommit (0.00s)
=== RUN   TestReplicaContains
--- PASS: TestReplicaContains (0.01s)
=== RUN   TestReplicaReadConsistency
--- PASS: TestReplicaReadConsistency (0.09s)
=== RUN   TestBehaviorDuringLeaseTransfer
I170517 07:31:49.142584 3438 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:49.149778 3440 storage/replica_command.go:177  [s1,r1/1:/M{in-ax}] test injecting error: storage/replica_test.go:535: injected transfer error
--- PASS: TestBehaviorDuringLeaseTransfer (0.10s)
=== RUN   TestApplyCmdLeaseError
--- PASS: TestApplyCmdLeaseError (0.04s)
=== RUN   TestReplicaRangeBoundsChecking
I170517 07:31:49.277565 3785 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaRangeBoundsChecking (0.08s)
=== RUN   TestReplicaLease
I170517 07:31:49.359116 3882 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaLease (0.05s)
=== RUN   TestReplicaNotLeaseHolderError
I170517 07:31:49.419898 3904 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaNotLeaseHolderError (0.03s)
=== RUN   TestReplicaLeaseCounters
I170517 07:31:49.466652 3697 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaLeaseCounters (0.10s)
=== RUN   TestReplicaGossipConfigsOnLease
I170517 07:31:49.543888 4117 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaGossipConfigsOnLease (0.09s)
=== RUN   TestReplicaTSCacheLowWaterOnLease
I170517 07:31:49.638633 4122 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaTSCacheLowWaterOnLease (0.04s)
=== RUN   TestReplicaLeaseRejectUnknownRaftNodeID
I170517 07:31:49.678615 4357 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:49.684879 4357 util/stop/stopper.go:505  quiescing; tasks left:
3      gossip/infostore.go:301
--- PASS: TestReplicaLeaseRejectUnknownRaftNodeID (0.03s)
=== RUN   TestReplicaDrainLease
--- PASS: TestReplicaDrainLease (0.08s)
=== RUN   TestReplicaGossipFirstRange
I170517 07:31:49.801399 4191 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestReplicaGossipFirstRange (0.03s)
=== RUN   TestReplicaGossipAllConfigs
I170517 07:31:49.821567 4632 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:49.825682 4632 util/stop/stopper.go:505  quiescing; tasks left:
3      gossip/infostore.go:301
I170517 07:31:49.828218 4632 util/stop/stopper.go:505  quiescing; tasks left:
2      gossip/infostore.go:301
I170517 07:31:49.828344 4632 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestReplicaGossipAllConfigs (0.08s)
=== RUN   TestReplicaNoGossipConfig
I170517 07:31:49.922041 4633 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaNoGossipConfig (0.10s)
=== RUN   TestReplicaNoGossipFromNonLeader
I170517 07:31:50.010467 4712 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaNoGossipFromNonLeader (0.10s)
=== RUN   TestOptimizePuts
I170517 07:31:50.108399 4832 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestOptimizePuts (0.07s)
=== RUN   TestAcquireLease
=== RUN   TestAcquireLease/#00
I170517 07:31:50.166308 5022 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestAcquireLease/#01
I170517 07:31:50.203114 4817 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestAcquireLease (0.08s)
    --- PASS: TestAcquireLease/#00 (0.02s)
    --- PASS: TestAcquireLease/#01 (0.04s)
=== RUN   TestLeaseConcurrent
I170517 07:31:50.269165 5181 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:50.305422 5181 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestLeaseConcurrent (0.09s)
=== RUN   TestReplicaUpdateTSCache
I170517 07:31:50.332475 4909 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaUpdateTSCache (0.04s)
=== RUN   TestReplicaCommandQueue
=== RUN   TestReplicaCommandQueue/read-read
I170517 07:31:50.374188 5373 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/read-read-local
I170517 07:31:50.393737 5571 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/read-read-noop
I170517 07:31:50.412364 5661 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/read-read-noop-local
I170517 07:31:50.429095 5771 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/read-read-addRead
=== RUN   TestReplicaCommandQueue/read-read-addRead-local
I170517 07:31:50.474931 5956 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/read-write
I170517 07:31:50.503438 6032 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/read-write-local
I170517 07:31:50.531151 6152 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:50.539675 6152 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/replica_test.go:2072
W170517 07:31:50.541370 6244 storage/replica.go:2296  [s1,r1/1:/M{in-ax}] shutdown cancellation after 0.0s of attempting command Put [/Local/Range/"key1-read-write-local",/Min)
=== RUN   TestReplicaCommandQueue/read-write-noop
I170517 07:31:50.565273 6246 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/read-write-noop-local
=== RUN   TestReplicaCommandQueue/read-write-addRead
I170517 07:31:50.612523 6342 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/read-write-addRead-local
I170517 07:31:50.641495 6376 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:50.654139 6376 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/replica_test.go:2072
1      gossip/infostore.go:301
W170517 07:31:50.654306 6642 storage/replica.go:2296  [s1,r1/1:/M{in-ax}] shutdown cancellation after 0.0s of attempting command Put [/Local/Range/"key1-read-write-addRead-local",/Min)
I170517 07:31:50.654601 6376 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
=== RUN   TestReplicaCommandQueue/write-read
I170517 07:31:50.668438 6455 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/write-read-local
I170517 07:31:50.691338 6745 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/write-read-noop
I170517 07:31:50.745598 6550 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/replica_test.go:2081
=== RUN   TestReplicaCommandQueue/write-read-noop-local
I170517 07:31:50.757928 6823 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/write-read-addWrite
I170517 07:31:50.797180 7021 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/write-read-addWrite-local
I170517 07:31:50.827803 7033 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/write-write
I170517 07:31:50.857869 6942 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/write-write-local
I170517 07:31:50.890477 7314 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/write-write-noop
I170517 07:31:50.911126 7305 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/write-write-noop-local
I170517 07:31:50.930865 7469 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueue/write-write-addWrite
=== RUN   TestReplicaCommandQueue/write-write-addWrite-local
--- PASS: TestReplicaCommandQueue (0.70s)
    --- PASS: TestReplicaCommandQueue/read-read (0.02s)
    --- PASS: TestReplicaCommandQueue/read-read-local (0.02s)
    --- PASS: TestReplicaCommandQueue/read-read-noop (0.02s)
    --- PASS: TestReplicaCommandQueue/read-read-noop-local (0.02s)
    --- PASS: TestReplicaCommandQueue/read-read-addRead (0.03s)
    --- PASS: TestReplicaCommandQueue/read-read-addRead-local (0.02s)
    --- PASS: TestReplicaCommandQueue/read-write (0.03s)
    --- PASS: TestReplicaCommandQueue/read-write-local (0.04s)
    --- PASS: TestReplicaCommandQueue/read-write-noop (0.03s)
    --- PASS: TestReplicaCommandQueue/read-write-noop-local (0.03s)
    --- PASS: TestReplicaCommandQueue/read-write-addRead (0.02s)
    --- PASS: TestReplicaCommandQueue/read-write-addRead-local (0.04s)
    --- PASS: TestReplicaCommandQueue/write-read (0.02s)
    --- PASS: TestReplicaCommandQueue/write-read-local (0.03s)
    --- PASS: TestReplicaCommandQueue/write-read-noop (0.03s)
    --- PASS: TestReplicaCommandQueue/write-read-noop-local (0.02s)
    --- PASS: TestReplicaCommandQueue/write-read-addWrite (0.05s)
    --- PASS: TestReplicaCommandQueue/write-read-addWrite-local (0.02s)
    --- PASS: TestReplicaCommandQueue/write-write (0.04s)
    --- PASS: TestReplicaCommandQueue/write-write-local (0.02s)
    --- PASS: TestReplicaCommandQueue/write-write-noop (0.02s)
    --- PASS: TestReplicaCommandQueue/write-write-noop-local (0.04s)
    --- PASS: TestReplicaCommandQueue/write-write-addWrite (0.06s)
    --- PASS: TestReplicaCommandQueue/write-write-addWrite-local (0.02s)
=== RUN   TestReplicaCommandQueueInconsistent
I170517 07:31:51.066058 7589 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaCommandQueueInconsistent (0.09s)
=== RUN   TestReplicaCommandQueueCancellation
I170517 07:31:51.159148 7944 storage/replica_test.go:2237  starting to block header:<key:"one" > value:<raw_bytes:"\000\000\000\000\003" timestamp:<wall_time:0 logical:0 > > inline:false blind:false
W170517 07:31:51.159675 7945 storage/replica.go:1693  [s1,r1/1:/M{in-ax}] context canceled before command queue: Put ["one",/Min), Put ["two",/Min)
W170517 07:31:51.160469 7861 storage/replica.go:1739  [s1,r1/1:/M{in-ax}] context canceled while in command queue: Put ["one",/Min), Put ["two",/Min)
I170517 07:31:51.161795 7771 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/replica_test.go:2265
--- PASS: TestReplicaCommandQueueCancellation (0.03s)
=== RUN   TestReplicaCommandQueueSelfOverlap
I170517 07:31:51.183130 7969 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueueSelfOverlap/false,false
=== RUN   TestReplicaCommandQueueSelfOverlap/false,true
=== RUN   TestReplicaCommandQueueSelfOverlap/true,false
=== RUN   TestReplicaCommandQueueSelfOverlap/true,true
--- PASS: TestReplicaCommandQueueSelfOverlap (0.08s)
    --- PASS: TestReplicaCommandQueueSelfOverlap/false,false (0.00s)
    --- PASS: TestReplicaCommandQueueSelfOverlap/false,true (0.00s)
    --- PASS: TestReplicaCommandQueueSelfOverlap/true,false (0.00s)
    --- PASS: TestReplicaCommandQueueSelfOverlap/true,true (0.00s)
=== RUN   TestReplicaCommandQueueTimestampNonInterference
I170517 07:31:51.265307 8067 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:0}_writerTS:{WallTime:1_Logical:0}_key:[97]_readerFirst:true_interferes:true}
=== RUN   TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:0}_writerTS:{WallTime:1_Logical:0}_key:[98]_readerFirst:false_interferes:true}
=== RUN   TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:0}_writerTS:{WallTime:1_Logical:1}_key:[99]_readerFirst:true_interferes:false}
=== RUN   TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:0}_writerTS:{WallTime:1_Logical:1}_key:[100]_readerFirst:false_interferes:false}
=== RUN   TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:1}_writerTS:{WallTime:1_Logical:0}_key:[101]_readerFirst:true_interferes:true}
=== RUN   TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:1}_writerTS:{WallTime:1_Logical:0}_key:[102]_readerFirst:false_interferes:true}
=== RUN   TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:0}_writerTS:{WallTime:1_Logical:1}_key:[1_107_18_97_0_1_114_100_115_99]_readerFirst:true_interferes:true}
=== RUN   TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:0}_writerTS:{WallTime:1_Logical:1}_key:[1_107_18_98_0_1_114_100_115_99]_readerFirst:false_interferes:true}
--- PASS: TestReplicaCommandQueueTimestampNonInterference (0.11s)
    --- PASS: TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:0}_writerTS:{WallTime:1_Logical:0}_key:[97]_readerFirst:true_interferes:true} (0.01s)
    --- PASS: TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:0}_writerTS:{WallTime:1_Logical:0}_key:[98]_readerFirst:false_interferes:true} (0.01s)
    --- PASS: TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:0}_writerTS:{WallTime:1_Logical:1}_key:[99]_readerFirst:true_interferes:false} (0.00s)
    --- PASS: TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:0}_writerTS:{WallTime:1_Logical:1}_key:[100]_readerFirst:false_interferes:false} (0.00s)
    --- PASS: TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:1}_writerTS:{WallTime:1_Logical:0}_key:[101]_readerFirst:true_interferes:true} (0.01s)
    --- PASS: TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:1}_writerTS:{WallTime:1_Logical:0}_key:[102]_readerFirst:false_interferes:true} (0.01s)
    --- PASS: TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:0}_writerTS:{WallTime:1_Logical:1}_key:[1_107_18_97_0_1_114_100_115_99]_readerFirst:true_interferes:true} (0.01s)
    --- PASS: TestReplicaCommandQueueTimestampNonInterference/{readerTS:{WallTime:1_Logical:0}_writerTS:{WallTime:1_Logical:1}_key:[1_107_18_98_0_1_114_100_115_99]_readerFirst:false_interferes:true} (0.01s)
=== RUN   TestReplicaCommandQueueSplitDeclaresWrites
--- PASS: TestReplicaCommandQueueSplitDeclaresWrites (0.00s)
=== RUN   TestReplicaUseTSCache
I170517 07:31:51.398288 8052 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaUseTSCache (0.10s)
=== RUN   TestReplicaNoTSCacheInconsistent
I170517 07:31:51.480538 8260 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaNoTSCacheInconsistent (0.09s)
=== RUN   TestReplicaNoTSCacheUpdateOnFailure
--- PASS: TestReplicaNoTSCacheUpdateOnFailure (0.11s)
=== RUN   TestReplicaNoTimestampIncrementWithinTxn
--- PASS: TestReplicaNoTimestampIncrementWithinTxn (0.03s)
=== RUN   TestReplicaAbortCacheReadError
I170517 07:31:51.712786 8454 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
E170517 07:31:51.720306 8454 storage/replica.go:4559  [s1,r1/1:/M{in-ax}] stalling replica due to: could not read from abort cache: proto: illegal wireType 6
I170517 07:31:51.720725 8454 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestReplicaAbortCacheReadError (0.03s)
=== RUN   TestReplicaAbortCacheStoredTxnRetryError
I170517 07:31:51.753792 8361 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaAbortCacheStoredTxnRetryError (0.04s)
=== RUN   TestTransactionRetryLeavesIntents
I170517 07:31:51.792990 8472 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestTransactionRetryLeavesIntents (0.03s)
=== RUN   TestReplicaAbortCacheOnlyWithIntent
I170517 07:31:51.817296 8476 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:51.827902 8476 util/stop/stopper.go:505  quiescing; tasks left:
3      gossip/infostore.go:301
I170517 07:31:51.828068 8476 util/stop/stopper.go:505  quiescing; tasks left:
2      gossip/infostore.go:301
I170517 07:31:51.828163 8476 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestReplicaAbortCacheOnlyWithIntent (0.09s)
=== RUN   TestEndTransactionDeadline
I170517 07:31:51.928195 8922 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestEndTransactionDeadline (0.13s)
=== RUN   TestEndTransactionTxnSpanGCThreshold
I170517 07:31:52.037357 8907 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestEndTransactionTxnSpanGCThreshold (0.03s)
=== RUN   TestEndTransactionDeadline_1PC
I170517 07:31:52.084344 9094 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestEndTransactionDeadline_1PC (0.09s)
=== RUN   Test1PCTransactionWriteTimestamp
=== RUN   Test1PCTransactionWriteTimestamp/SNAPSHOT
=== RUN   Test1PCTransactionWriteTimestamp/SERIALIZABLE
--- PASS: Test1PCTransactionWriteTimestamp (0.09s)
    --- PASS: Test1PCTransactionWriteTimestamp/SNAPSHOT (0.00s)
    --- PASS: Test1PCTransactionWriteTimestamp/SERIALIZABLE (0.00s)
=== RUN   TestEndTransactionWithMalformedSplitTrigger
I170517 07:31:52.252115 9105 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
E170517 07:31:52.259567 9105 storage/replica.go:4559  [s1,r1/1:/M{in-ax}] stalling replica due to: range does not match splits: ("bar"-"foo") + ("foo"-/Max) != [n1,s1,r1/1:/M{in-ax}]
--- PASS: TestEndTransactionWithMalformedSplitTrigger (0.03s)
=== RUN   TestEndTransactionBeforeHeartbeat
I170517 07:31:52.282998 9374 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestEndTransactionBeforeHeartbeat (0.13s)
=== RUN   TestEndTransactionAfterHeartbeat
I170517 07:31:52.409095 9475 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestEndTransactionAfterHeartbeat (0.06s)
=== RUN   TestEndTransactionWithPushedTimestamp
I170517 07:31:52.466917 9469 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestEndTransactionWithPushedTimestamp (0.04s)
=== RUN   TestEndTransactionWithIncrementedEpoch
--- PASS: TestEndTransactionWithIncrementedEpoch (0.03s)
=== RUN   TestEndTransactionWithErrors
I170517 07:31:52.545829 9769 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestEndTransactionWithErrors (0.04s)
=== RUN   TestEndTransactionRollbackAbortedTransaction
I170517 07:31:52.583838 8739 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestEndTransactionRollbackAbortedTransaction (0.10s)
=== RUN   TestRaftRetryProtectionInTxn
--- PASS: TestRaftRetryProtectionInTxn (0.03s)
=== RUN   TestReplicaLaziness
I170517 07:31:52.715705 10023 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:52.733140 10023 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:52.748320 10023 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaLaziness (0.12s)
=== RUN   TestRaftRetryCantCommitIntents
I170517 07:31:52.828369 10024 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestRaftRetryCantCommitIntents/SERIALIZABLE
=== RUN   TestRaftRetryCantCommitIntents/SNAPSHOT
--- PASS: TestRaftRetryCantCommitIntents (0.09s)
    --- PASS: TestRaftRetryCantCommitIntents/SERIALIZABLE (0.01s)
    --- PASS: TestRaftRetryCantCommitIntents/SNAPSHOT (0.01s)
=== RUN   TestDuplicateBeginTransaction
--- PASS: TestDuplicateBeginTransaction (0.05s)
=== RUN   TestEndTransactionLocalGC
I170517 07:31:52.975097 10458 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:52.996010 10546 storage/replica_command.go:177  [s1,r1/1:{/Min-c}] test injecting error: boom
W170517 07:31:53.001601 10546 storage/intent_resolver.go:370  could not GC completed transaction anchored at "a": boom
I170517 07:31:53.001710 10562 storage/replica_command.go:177  [s1,r1/1:{/Min-c}] test injecting error: boom
W170517 07:31:53.001984 10562 storage/intent_resolver.go:370  could not GC completed transaction anchored at "a": boom
I170517 07:31:53.002642 10458 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
W170517 07:31:53.003339 10526 storage/replica.go:2296  [s1,r2/1:{c-/Max}] shutdown cancellation after 0.0s of attempting command ResolveIntent ["c",/Min)
W170517 07:31:53.003501 10526 storage/intent_resolver.go:327  [n1,s1,r1/1:{/Min-c}]: failed to resolve intents: result is ambiguous (server shutdown)
--- PASS: TestEndTransactionLocalGC (0.06s)
=== RUN   TestEndTransactionResolveOnlyLocalIntents
I170517 07:31:53.032155 10563 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:53.046729 10651 storage/replica_command.go:177  [s1,r2/1:{a�-/Max}] test injecting error: boom
W170517 07:31:53.047064 10651 storage/intent_resolver.go:327  [n1,s1,r1/1:{/Min-a�}]: failed to resolve intents: boom
--- PASS: TestEndTransactionResolveOnlyLocalIntents (0.11s)
=== RUN   TestEndTransactionDirectGC
I170517 07:31:53.139235 10652 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:53.156717 10652 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
I170517 07:31:53.175412 10652 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:53.246844 10652 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestEndTransactionDirectGC (0.16s)
=== RUN   TestEndTransactionDirectGCFailure
I170517 07:31:53.296080 10933 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:53.311536 10860 storage/replica_command.go:177  [s1,r2/1:{a�-/Max}] test injecting error: boom
I170517 07:31:53.312807 10933 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
W170517 07:31:53.313365 10860 storage/intent_resolver.go:327  [n1,s1,r1/1:{/Min-a�}]: failed to resolve intents: boom
--- PASS: TestEndTransactionDirectGCFailure (0.04s)
=== RUN   TestEndTransactionDirectGC_1PC
I170517 07:31:53.339202 11029 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:53.352595 11029 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestEndTransactionDirectGC_1PC (0.09s)
=== RUN   TestReplicaTransactionRequires1PC
I170517 07:31:53.424770 11133 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestReplicaTransactionRequires1PC/#00
=== RUN   TestReplicaTransactionRequires1PC/#01
I170517 07:31:53.432402 10955 storage/replica_command.go:177  [s1,r1/1:/M{in-ax}] test injecting error: storage/replica_test.go:4025: injected error
--- PASS: TestReplicaTransactionRequires1PC (0.09s)
    --- PASS: TestReplicaTransactionRequires1PC/#00 (0.00s)
    --- PASS: TestReplicaTransactionRequires1PC/#01 (0.00s)
=== RUN   TestReplicaEndTransactionWithRequire1PC
I170517 07:31:53.515895 11312 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaEndTransactionWithRequire1PC (0.04s)
=== RUN   TestReplicaResolveIntentNoWait
I170517 07:31:53.565752 11204 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:53.621060 11204 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
W170517 07:31:53.621478 11390 storage/replica.go:2296  [s1,r1/1:{/Min-aa}] shutdown cancellation after 0.0s of attempting command GC ["a","a\x00")
W170517 07:31:53.621567 11390 storage/intent_resolver.go:370  could not GC completed transaction anchored at "a": result is ambiguous (server shutdown)
--- PASS: TestReplicaResolveIntentNoWait (0.09s)
=== RUN   TestAbortCachePoisonOnResolve
I170517 07:31:53.653136 11391 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:53.705402 11391 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:53.745762 11391 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestAbortCachePoisonOnResolve (0.15s)
=== RUN   TestAbortCacheError
I170517 07:31:53.792088 11279 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestAbortCacheError (0.08s)
=== RUN   TestPushTxnBadKey
I170517 07:31:53.876588 11670 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:53.882211 11670 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestPushTxnBadKey (0.05s)
=== RUN   TestPushTxnAlreadyCommittedOrAborted
I170517 07:31:53.931454 11769 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestPushTxnAlreadyCommittedOrAborted (0.03s)
=== RUN   TestPushTxnUpgradeExistingTxn
I170517 07:31:53.969235 11958 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestPushTxnUpgradeExistingTxn (0.11s)
=== RUN   TestPushTxnQueryPusheeHasNewerVersion
I170517 07:31:54.067309 12226 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestPushTxnQueryPusheeHasNewerVersion (0.09s)
=== RUN   TestPushTxnHeartbeatTimeout
I170517 07:31:54.185090 12302 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestPushTxnHeartbeatTimeout (0.10s)
=== RUN   TestResolveIntentPushTxnReplyTxn
I170517 07:31:54.282888 12223 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:54.287888 12223 util/stop/stopper.go:505  quiescing; tasks left:
2      gossip/infostore.go:301
I170517 07:31:54.288118 12223 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestResolveIntentPushTxnReplyTxn (0.10s)
=== RUN   TestPushTxnPriorities
I170517 07:31:54.377057 12514 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestPushTxnPriorities (0.06s)
=== RUN   TestPushTxnPushTimestamp
--- PASS: TestPushTxnPushTimestamp (0.03s)
=== RUN   TestPushTxnPushTimestampAlreadyPushed
I170517 07:31:54.455212 11230 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestPushTxnPushTimestampAlreadyPushed (0.08s)
=== RUN   TestPushTxnSerializableRestart
I170517 07:31:54.531277 12405 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestPushTxnSerializableRestart (0.03s)
=== RUN   TestReplicaResolveIntentRange
I170517 07:31:54.570081 12501 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaResolveIntentRange (0.03s)
=== RUN   TestRangeStatsComputation
I170517 07:31:54.602067 12941 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestRangeStatsComputation (0.05s)
=== RUN   TestMerge
I170517 07:31:54.659369 11860 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestMerge (0.10s)
=== RUN   TestTruncateLog
I170517 07:31:54.765582 12849 storage/replica_command.go:1836  [s1,r1/1:/M{in-ax}] attempting to truncate raft logs for another range: r2. Normally this is due to a merge and can be ignored.
--- PASS: TestTruncateLog (0.05s)
=== RUN   TestConditionFailedError
I170517 07:31:54.788756 13202 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestConditionFailedError (0.03s)
=== RUN   TestReplicaSetsEqual
--- PASS: TestReplicaSetsEqual (0.01s)
=== RUN   TestAppliedIndex
I170517 07:31:54.830663 13283 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestAppliedIndex (0.04s)
=== RUN   TestReplicaCorruption
I170517 07:31:54.898661 12785 storage/replica_command.go:177  [s1,r1/1:/M{in-ax}] test injecting error: replica corruption (processed=false): boom
E170517 07:31:54.898770 12785 storage/replica.go:4559  [s1,r1/1:/M{in-ax}] stalling replica due to: boom
--- PASS: TestReplicaCorruption (0.09s)
=== RUN   TestChangeReplicasDuplicateError
--- PASS: TestChangeReplicasDuplicateError (0.07s)
=== RUN   TestReplicaDanglingMetaIntent
I170517 07:31:55.054822 13578 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:31:55.063109 13578 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/intent_resolver.go:259
I170517 07:31:55.067795 13578 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
W170517 07:31:58.063648 13390 storage/intent_resolver.go:313  [n1,s1,r1/1:/M{in-ax}]: failed to push during intent resolution: context deadline exceeded
I170517 07:31:58.090739 13578 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/intent_resolver.go:259
I170517 07:31:58.092120 13578 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
W170517 07:32:01.096033 13764 storage/intent_resolver.go:313  [n1,s1,r1/1:/M{in-ax}]: failed to push during intent resolution: context deadline exceeded
--- PASS: TestReplicaDanglingMetaIntent (6.11s)
=== RUN   TestReplicaLookupUseReverseScan
I170517 07:32:01.179873 13760 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:01.198564 13760 util/stop/stopper.go:505  quiescing; tasks left:
7      storage/intent_resolver.go:259
I170517 07:32:01.199122 13760 util/stop/stopper.go:505  quiescing; tasks left:
6      storage/intent_resolver.go:259
I170517 07:32:01.199269 13760 util/stop/stopper.go:505  quiescing; tasks left:
5      storage/intent_resolver.go:259
I170517 07:32:01.199422 13760 util/stop/stopper.go:505  quiescing; tasks left:
4      storage/intent_resolver.go:259
I170517 07:32:01.199556 13760 util/stop/stopper.go:505  quiescing; tasks left:
3      storage/intent_resolver.go:259
I170517 07:32:01.199681 13760 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/intent_resolver.go:259
I170517 07:32:01.199838 13760 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
W170517 07:32:01.200442 13853 storage/replica.go:2296  [s1,r1/1:/M{in-ax}] shutdown cancellation after 0.0s of attempting command PushTxn [/Min,/Min)
W170517 07:32:01.200542 13853 storage/intent_resolver.go:313  [n1,s1,r1/1:/M{in-ax}]: failed to push during intent resolution: result is ambiguous (server shutdown)
--- PASS: TestReplicaLookupUseReverseScan (0.14s)
=== RUN   TestRangeLookup
--- PASS: TestRangeLookup (0.03s)
=== RUN   TestRequestLeaderEncounterGroupDeleteError
--- PASS: TestRequestLeaderEncounterGroupDeleteError (0.10s)
=== RUN   TestIntentIntersect
--- PASS: TestIntentIntersect (0.01s)
=== RUN   TestBatchErrorWithIndex
I170517 07:32:01.433546 14053 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:01.439261 14053 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestBatchErrorWithIndex (0.03s)
=== RUN   TestReplicaLoadSystemConfigSpanIntent
I170517 07:32:01.460510 14156 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaLoadSystemConfigSpanIntent (0.05s)
=== RUN   TestReplicaDestroy
I170517 07:32:01.533941 14263 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:01.537766 14263 storage/store.go:2139  removing replica
I170517 07:32:01.538193 14263 storage/store.go:2139  removing replica
I170517 07:32:01.538599 14263 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
--- PASS: TestReplicaDestroy (0.05s)
=== RUN   TestEntries
I170517 07:32:01.575194 14252 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestEntries (0.08s)
=== RUN   TestTerm
I170517 07:32:01.650363 14254 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestTerm (0.09s)
=== RUN   TestGCIncorrectRange
W170517 07:32:01.765509 14511 util/hlc/hlc.go:214  remote wall time is too far ahead (2ns) to be trustworthy - updating anyway
--- PASS: TestGCIncorrectRange (0.04s)
=== RUN   TestReplicaCancelRaft
I170517 07:32:01.790988 14520 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
W170517 07:32:01.795768 14520 storage/replica.go:1693  context canceled before command queue: Get ["acdfg",/Min)
I170517 07:32:01.795959 14520 util/stop/stopper.go:505  quiescing; tasks left:
2      gossip/infostore.go:301
I170517 07:32:01.796108 14520 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
I170517 07:32:01.828635 14520 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
W170517 07:32:01.835898 14520 storage/replica.go:2283  context cancellation after 0.0s of attempting command Get ["acdfg",/Min)
I170517 07:32:01.835996 14520 util/stop/stopper.go:505  quiescing; tasks left:
2      gossip/infostore.go:301
I170517 07:32:01.836116 14520 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestReplicaCancelRaft (0.14s)
=== RUN   TestReplicaTryAbandon
I170517 07:32:01.924683 14636 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
W170517 07:32:01.931188 14636 storage/replica.go:2283  context cancellation after 0.0s of attempting command Put ["acdfg",/Min)
--- PASS: TestReplicaTryAbandon (0.08s)
=== RUN   TestComputeChecksumVersioning
I170517 07:32:02.003942 14873 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
E170517 07:32:02.010796 14873 storage/replica_command.go:2282  Incompatible versions: e=2, v=3
--- PASS: TestComputeChecksumVersioning (0.06s)
=== RUN   TestNewReplicaCorruptionError
--- PASS: TestNewReplicaCorruptionError (0.01s)
=== RUN   TestDiffRange
--- PASS: TestDiffRange (0.01s)
=== RUN   TestSyncSnapshot
I170517 07:32:02.083790 14711 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:02.090124 14711 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestSyncSnapshot (0.03s)
=== RUN   TestReplicaIDChangePending
I170517 07:32:02.114008 15093 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:02.119838 15093 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestReplicaIDChangePending (0.03s)
=== RUN   TestSetReplicaID
I170517 07:32:02.152918 14541 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestSetReplicaID/#00
=== RUN   TestSetReplicaID/#01
=== RUN   TestSetReplicaID/#02
=== RUN   TestSetReplicaID/#03
=== RUN   TestSetReplicaID/#04
--- PASS: TestSetReplicaID (0.03s)
    --- PASS: TestSetReplicaID/#00 (0.00s)
    --- PASS: TestSetReplicaID/#01 (0.00s)
    --- PASS: TestSetReplicaID/#02 (0.00s)
    --- PASS: TestSetReplicaID/#03 (0.00s)
    --- PASS: TestSetReplicaID/#04 (0.00s)
=== RUN   TestReplicaRetryRaftProposal
I170517 07:32:02.218146 15075 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:02.227267 15075 storage/replica_test.go:6688  test begins
--- PASS: TestReplicaRetryRaftProposal (0.07s)
=== RUN   TestReplicaCancelRaftCommandProgress
I170517 07:32:02.265902 14997 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:02.280356 14997 storage/replica_test.go:6766  abandoning command 1
I170517 07:32:02.280765 14997 storage/replica_test.go:6766  abandoning command 3
I170517 07:32:02.280919 14997 storage/replica_test.go:6766  abandoning command 4
I170517 07:32:02.281483 14997 storage/replica_test.go:6766  abandoning command 6
I170517 07:32:02.282404 14997 storage/replica_test.go:6766  abandoning command 8
--- PASS: TestReplicaCancelRaftCommandProgress (0.05s)
=== RUN   TestReplicaBurstPendingCommandsAndRepropose
I170517 07:32:02.327776 15294 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaBurstPendingCommandsAndRepropose (0.12s)
=== RUN   TestReplicaRefreshPendingCommandsTicks
--- PASS: TestReplicaRefreshPendingCommandsTicks (0.04s)
=== RUN   TestAmbiguousResultErrorOnRetry
I170517 07:32:02.479436 15457 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
=== RUN   TestAmbiguousResultErrorOnRetry/non-txn-put
=== RUN   TestAmbiguousResultErrorOnRetry/1PC-txn
--- PASS: TestAmbiguousResultErrorOnRetry (0.04s)
    --- PASS: TestAmbiguousResultErrorOnRetry/non-txn-put (0.00s)
    --- PASS: TestAmbiguousResultErrorOnRetry/1PC-txn (0.01s)
=== RUN   TestCommandTimeThreshold
I170517 07:32:02.514660 15633 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
W170517 07:32:02.524656 15736 util/hlc/hlc.go:214  remote wall time is too far ahead (3ns) to be trustworthy - updating anyway
--- PASS: TestCommandTimeThreshold (0.09s)
=== RUN   TestDeprecatedRequests
--- PASS: TestDeprecatedRequests (0.08s)
=== RUN   TestReplicaTimestampCacheBumpNotLost
I170517 07:32:02.686380 15744 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReplicaTimestampCacheBumpNotLost (0.04s)
=== RUN   TestReplicaEvaluationNotTxnMutation
I170517 07:32:02.723457 15808 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:02.729104 15808 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestReplicaEvaluationNotTxnMutation (0.08s)
=== RUN   TestReplicaMetrics
=== RUN   TestReplicaMetrics/#00
=== RUN   TestReplicaMetrics/#01
=== RUN   TestReplicaMetrics/#02
=== RUN   TestReplicaMetrics/#03
=== RUN   TestReplicaMetrics/#04
=== RUN   TestReplicaMetrics/#05
=== RUN   TestReplicaMetrics/#06
=== RUN   TestReplicaMetrics/#07
=== RUN   TestReplicaMetrics/#08
=== RUN   TestReplicaMetrics/#09
=== RUN   TestReplicaMetrics/#10
=== RUN   TestReplicaMetrics/#11
=== RUN   TestReplicaMetrics/#12
=== RUN   TestReplicaMetrics/#13
=== RUN   TestReplicaMetrics/#14
=== RUN   TestReplicaMetrics/#15
=== RUN   TestReplicaMetrics/#16
=== RUN   TestReplicaMetrics/#17
--- PASS: TestReplicaMetrics (0.01s)
    --- PASS: TestReplicaMetrics/#00 (0.00s)
    --- PASS: TestReplicaMetrics/#01 (0.00s)
    --- PASS: TestReplicaMetrics/#02 (0.00s)
    --- PASS: TestReplicaMetrics/#03 (0.00s)
    --- PASS: TestReplicaMetrics/#04 (0.00s)
    --- PASS: TestReplicaMetrics/#05 (0.00s)
    --- PASS: TestReplicaMetrics/#06 (0.00s)
    --- PASS: TestReplicaMetrics/#07 (0.00s)
    --- PASS: TestReplicaMetrics/#08 (0.00s)
    --- PASS: TestReplicaMetrics/#09 (0.00s)
    --- PASS: TestReplicaMetrics/#10 (0.00s)
    --- PASS: TestReplicaMetrics/#11 (0.00s)
    --- PASS: TestReplicaMetrics/#12 (0.00s)
    --- PASS: TestReplicaMetrics/#13 (0.00s)
    --- PASS: TestReplicaMetrics/#14 (0.00s)
    --- PASS: TestReplicaMetrics/#15 (0.00s)
    --- PASS: TestReplicaMetrics/#16 (0.00s)
    --- PASS: TestReplicaMetrics/#17 (0.00s)
=== RUN   TestCancelPendingCommands
I170517 07:32:02.823033 16119 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestCancelPendingCommands (0.03s)
=== RUN   TestMakeTimestampCacheRequest
=== RUN   TestMakeTimestampCacheRequest/#00
=== RUN   TestMakeTimestampCacheRequest/#01
=== RUN   TestMakeTimestampCacheRequest/#02
=== RUN   TestMakeTimestampCacheRequest/#03
=== RUN   TestMakeTimestampCacheRequest/#04
=== RUN   TestMakeTimestampCacheRequest/#05
=== RUN   TestMakeTimestampCacheRequest/#06
=== RUN   TestMakeTimestampCacheRequest/#07
=== RUN   TestMakeTimestampCacheRequest/#08
=== RUN   TestMakeTimestampCacheRequest/#09
--- PASS: TestMakeTimestampCacheRequest (0.01s)
    --- PASS: TestMakeTimestampCacheRequest/#00 (0.00s)
    --- PASS: TestMakeTimestampCacheRequest/#01 (0.00s)
    --- PASS: TestMakeTimestampCacheRequest/#02 (0.00s)
    --- PASS: TestMakeTimestampCacheRequest/#03 (0.00s)
    --- PASS: TestMakeTimestampCacheRequest/#04 (0.00s)
    --- PASS: TestMakeTimestampCacheRequest/#05 (0.00s)
    --- PASS: TestMakeTimestampCacheRequest/#06 (0.00s)
    --- PASS: TestMakeTimestampCacheRequest/#07 (0.00s)
    --- PASS: TestMakeTimestampCacheRequest/#08 (0.00s)
    --- PASS: TestMakeTimestampCacheRequest/#09 (0.00s)
=== RUN   TestCommandTooLarge
I170517 07:32:02.857941 16202 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:02.864571 16202 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestCommandTooLarge (0.03s)
=== RUN   TestOnlyValid
=== RUN   TestOnlyValid/0,0
=== RUN   TestOnlyValid/1,0
=== RUN   TestOnlyValid/0,1
=== RUN   TestOnlyValid/1,1
=== RUN   TestOnlyValid/2,0
=== RUN   TestOnlyValid/2,1
=== RUN   TestOnlyValid/2,2
=== RUN   TestOnlyValid/1,2
=== RUN   TestOnlyValid/0,2
--- PASS: TestOnlyValid (0.01s)
    --- PASS: TestOnlyValid/0,0 (0.00s)
    --- PASS: TestOnlyValid/1,0 (0.00s)
    --- PASS: TestOnlyValid/0,1 (0.00s)
    --- PASS: TestOnlyValid/1,1 (0.00s)
    --- PASS: TestOnlyValid/2,0 (0.00s)
    --- PASS: TestOnlyValid/2,1 (0.00s)
    --- PASS: TestOnlyValid/2,2 (0.00s)
    --- PASS: TestOnlyValid/1,2 (0.00s)
    --- PASS: TestOnlyValid/0,2 (0.00s)
=== RUN   TestCandidateSelection
=== RUN   TestCandidateSelection/best-0:0
=== RUN   TestCandidateSelection/worst-0:0
=== RUN   TestCandidateSelection/good-0:0
=== RUN   TestCandidateSelection/bad-0:0
=== RUN   TestCandidateSelection/best-0:0,0:1
=== RUN   TestCandidateSelection/worst-0:0,0:1
=== RUN   TestCandidateSelection/good-0:0,0:1
=== RUN   TestCandidateSelection/bad-0:0,0:1
=== RUN   TestCandidateSelection/best-0:0,0:1,0:2
=== RUN   TestCandidateSelection/worst-0:0,0:1,0:2
=== RUN   TestCandidateSelection/good-0:0,0:1,0:2
=== RUN   TestCandidateSelection/bad-0:0,0:1,0:2
=== RUN   TestCandidateSelection/best-1:0,0:1
=== RUN   TestCandidateSelection/worst-1:0,0:1
=== RUN   TestCandidateSelection/good-1:0,0:1
=== RUN   TestCandidateSelection/bad-1:0,0:1
=== RUN   TestCandidateSelection/best-1:0,0:1,0:2
=== RUN   TestCandidateSelection/worst-1:0,0:1,0:2
=== RUN   TestCandidateSelection/good-1:0,0:1,0:2
=== RUN   TestCandidateSelection/bad-1:0,0:1,0:2
=== RUN   TestCandidateSelection/best-1:0,1:1,0:2
=== RUN   TestCandidateSelection/worst-1:0,1:1,0:2
=== RUN   TestCandidateSelection/good-1:0,1:1,0:2
=== RUN   TestCandidateSelection/bad-1:0,1:1,0:2
=== RUN   TestCandidateSelection/best-1:0,1:1,0:2,0:3
=== RUN   TestCandidateSelection/worst-1:0,1:1,0:2,0:3
=== RUN   TestCandidateSelection/good-1:0,1:1,0:2,0:3
=== RUN   TestCandidateSelection/bad-1:0,1:1,0:2,0:3
--- PASS: TestCandidateSelection (0.02s)
    --- PASS: TestCandidateSelection/best-0:0 (0.00s)
    --- PASS: TestCandidateSelection/worst-0:0 (0.00s)
    --- PASS: TestCandidateSelection/good-0:0 (0.00s)
    --- PASS: TestCandidateSelection/bad-0:0 (0.00s)
    --- PASS: TestCandidateSelection/best-0:0,0:1 (0.00s)
    --- PASS: TestCandidateSelection/worst-0:0,0:1 (0.00s)
    --- PASS: TestCandidateSelection/good-0:0,0:1 (0.00s)
    --- PASS: TestCandidateSelection/bad-0:0,0:1 (0.00s)
    --- PASS: TestCandidateSelection/best-0:0,0:1,0:2 (0.00s)
    --- PASS: TestCandidateSelection/worst-0:0,0:1,0:2 (0.00s)
    --- PASS: TestCandidateSelection/good-0:0,0:1,0:2 (0.00s)
    --- PASS: TestCandidateSelection/bad-0:0,0:1,0:2 (0.00s)
    --- PASS: TestCandidateSelection/best-1:0,0:1 (0.00s)
    --- PASS: TestCandidateSelection/worst-1:0,0:1 (0.00s)
    --- PASS: TestCandidateSelection/good-1:0,0:1 (0.00s)
    --- PASS: TestCandidateSelection/bad-1:0,0:1 (0.00s)
    --- PASS: TestCandidateSelection/best-1:0,0:1,0:2 (0.00s)
    --- PASS: TestCandidateSelection/worst-1:0,0:1,0:2 (0.00s)
    --- PASS: TestCandidateSelection/good-1:0,0:1,0:2 (0.00s)
    --- PASS: TestCandidateSelection/bad-1:0,0:1,0:2 (0.00s)
    --- PASS: TestCandidateSelection/best-1:0,1:1,0:2 (0.00s)
    --- PASS: TestCandidateSelection/worst-1:0,1:1,0:2 (0.00s)
    --- PASS: TestCandidateSelection/good-1:0,1:1,0:2 (0.00s)
    --- PASS: TestCandidateSelection/bad-1:0,1:1,0:2 (0.00s)
    --- PASS: TestCandidateSelection/best-1:0,1:1,0:2,0:3 (0.00s)
    --- PASS: TestCandidateSelection/worst-1:0,1:1,0:2,0:3 (0.00s)
    --- PASS: TestCandidateSelection/good-1:0,1:1,0:2,0:3 (0.00s)
    --- PASS: TestCandidateSelection/bad-1:0,1:1,0:2,0:3 (0.00s)
=== RUN   TestBetterThan
--- PASS: TestBetterThan (0.04s)
=== RUN   TestPreexistingReplicaCheck
--- PASS: TestPreexistingReplicaCheck (0.02s)
=== RUN   TestConstraintCheck
=== RUN   TestConstraintCheck/required_constraint
=== RUN   TestConstraintCheck/required_locality_constraints
=== RUN   TestConstraintCheck/prohibited_constraints
=== RUN   TestConstraintCheck/prohibited_locality_constraints
=== RUN   TestConstraintCheck/positive_constraints
=== RUN   TestConstraintCheck/positive_locality_constraints
--- PASS: TestConstraintCheck (0.01s)
    --- PASS: TestConstraintCheck/required_constraint (0.00s)
    --- PASS: TestConstraintCheck/required_locality_constraints (0.00s)
    --- PASS: TestConstraintCheck/prohibited_constraints (0.00s)
    --- PASS: TestConstraintCheck/prohibited_locality_constraints (0.00s)
    --- PASS: TestConstraintCheck/positive_constraints (0.00s)
    --- PASS: TestConstraintCheck/positive_locality_constraints (0.00s)
=== RUN   TestDiversityScore
=== RUN   TestDiversityScore/no_existing_replicas
=== RUN   TestDiversityScore/one_existing_replicas
=== RUN   TestDiversityScore/two_existing_replicas
--- PASS: TestDiversityScore (0.01s)
    --- PASS: TestDiversityScore/no_existing_replicas (0.00s)
    --- PASS: TestDiversityScore/one_existing_replicas (0.00s)
    --- PASS: TestDiversityScore/two_existing_replicas (0.00s)
=== RUN   TestDiversityRemovalScore
=== RUN   TestDiversityRemovalScore/four_existing_replicas
=== RUN   TestDiversityRemovalScore/three_existing_replicas_-_testStoreUSa15
=== RUN   TestDiversityRemovalScore/three_existing_replicas_-_testStoreUSa1
=== RUN   TestDiversityRemovalScore/three_existing_replicas_-_testStoreUSb
=== RUN   TestDiversityRemovalScore/three_existing_replicas_-_testStoreEurope
--- PASS: TestDiversityRemovalScore (0.01s)
    --- PASS: TestDiversityRemovalScore/four_existing_replicas (0.00s)
    --- PASS: TestDiversityRemovalScore/three_existing_replicas_-_testStoreUSa15 (0.00s)
    --- PASS: TestDiversityRemovalScore/three_existing_replicas_-_testStoreUSa1 (0.00s)
    --- PASS: TestDiversityRemovalScore/three_existing_replicas_-_testStoreUSb (0.00s)
    --- PASS: TestDiversityRemovalScore/three_existing_replicas_-_testStoreEurope (0.00s)
=== RUN   TestMaxCapacity
--- PASS: TestMaxCapacity (0.01s)
=== RUN   TestScannerAddToQueues
--- PASS: TestScannerAddToQueues (0.03s)
=== RUN   TestScannerTiming
I170517 07:32:03.126357 16370 storage/scanner_test.go:257  0: average scan: 15.26714ms
I170517 07:32:03.226954 16370 storage/scanner_test.go:257  1: average scan: 25.220169ms
--- PASS: TestScannerTiming (0.21s)
=== RUN   TestScannerPaceInterval
--- PASS: TestScannerPaceInterval (0.00s)
=== RUN   TestScannerDisabled
--- PASS: TestScannerDisabled (0.04s)
=== RUN   TestScannerDisabledWithZeroInterval
--- PASS: TestScannerDisabledWithZeroInterval (0.01s)
=== RUN   TestScannerEmptyRangeSet
--- PASS: TestScannerEmptyRangeSet (0.01s)
=== RUN   TestRangeIDChunk
--- PASS: TestRangeIDChunk (0.01s)
=== RUN   TestRangeIDQueue
--- PASS: TestRangeIDQueue (0.01s)
=== RUN   TestSchedulerLoop
--- PASS: TestSchedulerLoop (0.01s)
=== RUN   TestSchedulerBuffering
--- PASS: TestSchedulerBuffering (0.07s)
=== RUN   TestSpanSetGetSpansScope
--- PASS: TestSpanSetGetSpansScope (0.02s)
=== RUN   TestSpanSetCheckAllowedBoundaries
--- PASS: TestSpanSetCheckAllowedBoundaries (0.01s)
=== RUN   TestSpanSetWriteImpliesRead
--- PASS: TestSpanSetWriteImpliesRead (0.02s)
=== RUN   TestSpanSetBatch
--- PASS: TestSpanSetBatch (0.02s)
=== RUN   TestSplitQueueShouldQueue
--- PASS: TestSplitQueueShouldQueue (0.04s)
=== RUN   TestRangeStatsEmpty
I170517 07:32:03.536902 16368 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestRangeStatsEmpty (0.02s)
=== RUN   TestRangeStatsInit
--- PASS: TestRangeStatsInit (0.05s)
=== RUN   TestStorePoolGossipUpdate
--- PASS: TestStorePoolGossipUpdate (0.00s)
=== RUN   TestStorePoolGetStoreList
--- PASS: TestStorePoolGetStoreList (0.01s)
=== RUN   TestStorePoolGetStoreDetails
--- PASS: TestStorePoolGetStoreDetails (0.02s)
=== RUN   TestStorePoolFindDeadReplicas
--- PASS: TestStorePoolFindDeadReplicas (0.01s)
=== RUN   TestStorePoolDefaultState
--- PASS: TestStorePoolDefaultState (0.01s)
=== RUN   TestStorePoolThrottle
--- PASS: TestStorePoolThrottle (0.01s)
=== RUN   TestGetLocalities
--- PASS: TestGetLocalities (0.01s)
=== RUN   TestStoreInitAndBootstrap
--- PASS: TestStoreInitAndBootstrap (0.09s)
=== RUN   TestBootstrapOfNonEmptyStore
--- PASS: TestBootstrapOfNonEmptyStore (0.06s)
=== RUN   TestStoreAddRemoveRanges
I170517 07:32:03.844573 16683 storage/store.go:2139  removing replica
I170517 07:32:03.844858 16683 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:03.845867 16683 storage/store.go:2139  removing replica
--- PASS: TestStoreAddRemoveRanges (0.04s)
=== RUN   TestReplicasByKey
--- PASS: TestReplicasByKey (0.05s)
=== RUN   TestStoreRemoveReplicaOldDescriptor
I170517 07:32:03.946123 16777 storage/store.go:2139  removing replica
I170517 07:32:03.946452 16777 storage/store.go:2139  removing replica
I170517 07:32:03.946790 16777 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
--- PASS: TestStoreRemoveReplicaOldDescriptor (0.05s)
=== RUN   TestStoreRemoveReplicaDestroy
I170517 07:32:03.985823 17014 storage/store.go:2139  removing replica
I170517 07:32:03.986138 17014 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
--- PASS: TestStoreRemoveReplicaDestroy (0.04s)
=== RUN   TestStoreReplicaVisitor
I170517 07:32:04.022008 16671 storage/store.go:2139  removing replica
I170517 07:32:04.022379 16671 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
--- PASS: TestStoreReplicaVisitor (0.05s)
=== RUN   TestHasOverlappingReplica
I170517 07:32:04.069809 15537 storage/store.go:2139  removing replica
I170517 07:32:04.070127 15537 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
--- PASS: TestHasOverlappingReplica (0.03s)
=== RUN   TestProcessRangeDescriptorUpdate
I170517 07:32:04.129039 17299 storage/store.go:2139  removing replica
I170517 07:32:04.129358 17299 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
--- PASS: TestProcessRangeDescriptorUpdate (0.06s)
=== RUN   TestStoreSend
--- PASS: TestStoreSend (0.03s)
=== RUN   TestStoreObservedTimestamp
I170517 07:32:04.195143 17508 storage/replica_command.go:177  [s1,r1/1:/M{in-ax}] test injecting error: boom
--- PASS: TestStoreObservedTimestamp (0.13s)
=== RUN   TestStoreAnnotateNow
I170517 07:32:04.321335 17684 storage/replica_command.go:177  [s1,r1/1:/M{in-ax}] test injecting error: boom
I170517 07:32:04.401883 17684 storage/replica_command.go:177  [s1,r1/1:/M{in-ax}] test injecting error: boom
--- PASS: TestStoreAnnotateNow (0.15s)
=== RUN   TestStoreExecuteNoop
--- PASS: TestStoreExecuteNoop (0.10s)
=== RUN   TestStoreVerifyKeys
--- PASS: TestStoreVerifyKeys (0.04s)
=== RUN   TestStoreSendUpdateTime
--- PASS: TestStoreSendUpdateTime (0.08s)
=== RUN   TestStoreSendWithZeroTime
I170517 07:32:04.701874 15559 util/stop/stopper.go:505  quiescing; tasks left:
2      gossip/infostore.go:301
I170517 07:32:04.702070 15559 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestStoreSendWithZeroTime (0.03s)
=== RUN   TestStoreSendWithClockOffset
I170517 07:32:04.734143 18046 util/stop/stopper.go:505  quiescing; tasks left:
2      gossip/infostore.go:301
I170517 07:32:04.734278 18046 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestStoreSendWithClockOffset (0.07s)
=== RUN   TestStoreSendBadRange
--- PASS: TestStoreSendBadRange (0.09s)
=== RUN   TestStoreSendOutOfRange
--- PASS: TestStoreSendOutOfRange (0.08s)
=== RUN   TestStoreRangeIDAllocation
--- PASS: TestStoreRangeIDAllocation (0.03s)
=== RUN   TestStoreReplicasByKey
I170517 07:32:05.024750 18675 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/id_alloc.go:108
--- PASS: TestStoreReplicasByKey (0.12s)
=== RUN   TestStoreSetRangesMaxBytes
--- PASS: TestStoreSetRangesMaxBytes (0.08s)
=== RUN   TestStoreLongTxnStarvation
--- PASS: TestStoreLongTxnStarvation (0.11s)
=== RUN   TestStoreResolveWriteIntent
--- PASS: TestStoreResolveWriteIntent (0.07s)
=== RUN   TestStoreResolveWriteIntentRollback
--- PASS: TestStoreResolveWriteIntentRollback (0.06s)
=== RUN   TestStoreResolveWriteIntentPushOnRead
--- PASS: TestStoreResolveWriteIntentPushOnRead (0.12s)
	store_test.go:1395: 0: unsetting WriteTooOld flag as a hack to keep this test passing; should address the TODO
	store_test.go:1395: 1: unsetting WriteTooOld flag as a hack to keep this test passing; should address the TODO
	store_test.go:1395: 2: unsetting WriteTooOld flag as a hack to keep this test passing; should address the TODO
	store_test.go:1395: 3: unsetting WriteTooOld flag as a hack to keep this test passing; should address the TODO
=== RUN   TestStoreResolveWriteIntentSnapshotIsolation
--- PASS: TestStoreResolveWriteIntentSnapshotIsolation (0.07s)
=== RUN   TestStoreResolveWriteIntentNoTxn
--- PASS: TestStoreResolveWriteIntentNoTxn (0.03s)
=== RUN   TestStoreReadInconsistent
I170517 07:32:05.687735 19390 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:05.747833 19390 util/stop/stopper.go:505  quiescing; tasks left:
3      storage/intent_resolver.go:259
I170517 07:32:05.748012 19390 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/intent_resolver.go:259
W170517 07:32:08.702352 19579 storage/intent_resolver.go:313  [n1,s1,r1/1:/M{in-ax}]: failed to push during intent resolution: context deadline exceeded
I170517 07:32:08.702519 19390 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
W170517 07:32:08.736756 19588 storage/intent_resolver.go:313  [n1,s1,r1/1:/M{in-ax}]: failed to push during intent resolution: context deadline exceeded
--- PASS: TestStoreReadInconsistent (3.10s)
=== RUN   TestStoreScanIntents
I170517 07:32:08.898707 19604 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/intent_resolver.go:259
W170517 07:32:09.687476 19336 storage/intent_resolver.go:313  [n1,s1,r1/1:/M{in-ax}]: failed to push during intent resolution: [NotLeaseHolderError] r1: replica (n1,s1):1 not lease holder; lease holder unknown
I170517 07:32:09.687624 19604 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
W170517 07:32:09.689401 19338 storage/intent_resolver.go:313  [n1,s1,r1/1:/M{in-ax}]: failed to push during intent resolution: [NotLeaseHolderError] r1: replica (n1,s1):1 not lease holder; lease holder unknown
--- PASS: TestStoreScanIntents (0.98s)
=== RUN   TestStoreScanInconsistentResolvesIntents
--- PASS: TestStoreScanInconsistentResolvesIntents (0.38s)
=== RUN   TestStoreBadRequests
--- PASS: TestStoreBadRequests (0.06s)
=== RUN   TestMaybeRemove
I170517 07:32:10.205318 18385 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:10.226976 18385 storage/store.go:2139  removing replica
I170517 07:32:10.227355 18385 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
--- PASS: TestMaybeRemove (0.09s)
=== RUN   TestStoreGCThreshold
I170517 07:32:10.303093 20018 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestStoreGCThreshold (0.08s)
=== RUN   TestStoreRangePlaceholders
I170517 07:32:10.378578 20093 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:10.383603 20093 storage/store.go:2139  removing replica
I170517 07:32:10.383927 20093 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
--- PASS: TestStoreRangePlaceholders (0.11s)
=== RUN   TestStoreRemovePlaceholderOnError
I170517 07:32:10.487630 20028 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:10.493481 20028 storage/store.go:2139  removing replica
I170517 07:32:10.494029 20028 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
--- PASS: TestStoreRemovePlaceholderOnError (0.04s)
=== RUN   TestStoreRemovePlaceholderOnRaftIgnored
I170517 07:32:10.533055 20236 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:10.536510 20236 storage/store.go:2139  removing replica
I170517 07:32:10.536829 20236 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
W170517 07:32:10.538287 20236 storage/replica.go:3379  [s1,r1/2:{-}] failed to look up recipient replica 0 in r1 while sending MsgAppResp: replica 0 not present in (n2,s2):2, []
--- PASS: TestStoreRemovePlaceholderOnRaftIgnored (0.23s)
=== RUN   TestRemovedReplicaTombstone
=== RUN   TestRemovedReplicaTombstone/#00
I170517 07:32:10.806076 20191 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:10.815710 20482 storage/store.go:2139  removing replica
I170517 07:32:10.816990 20482 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
=== RUN   TestRemovedReplicaTombstone/#01
I170517 07:32:10.829991 20483 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:10.834381 20566 storage/store.go:2139  removing replica
I170517 07:32:10.835693 20566 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
=== RUN   TestRemovedReplicaTombstone/#02
I170517 07:32:10.856107 20650 storage/store.go:2139  removing replica
I170517 07:32:10.858136 20650 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
=== RUN   TestRemovedReplicaTombstone/#03
I170517 07:32:10.870577 20651 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:10.878025 20573 storage/store.go:2139  removing replica
I170517 07:32:10.879509 20573 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
=== RUN   TestRemovedReplicaTombstone/#04
I170517 07:32:10.917837 20007 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:10.925347 20465 storage/store.go:2139  removing replica
I170517 07:32:10.926737 20465 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
=== RUN   TestRemovedReplicaTombstone/#05
I170517 07:32:10.963341 20918 storage/store.go:2139  removing replica
I170517 07:32:10.971323 20918 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
=== RUN   TestRemovedReplicaTombstone/#06
I170517 07:32:10.987124 20946 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:10.991656 20809 storage/store.go:2139  removing replica
I170517 07:32:10.993002 20809 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
=== RUN   TestRemovedReplicaTombstone/#07
I170517 07:32:11.005657 20810 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:11.014298 21122 storage/store.go:2139  removing replica
I170517 07:32:11.015648 21122 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
=== RUN   TestRemovedReplicaTombstone/#08
I170517 07:32:11.036377 21208 storage/store.go:2139  removing replica
I170517 07:32:11.037727 21208 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
=== RUN   TestRemovedReplicaTombstone/#09
I170517 07:32:11.049947 21209 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:11.076514 21231 storage/store.go:2139  removing replica
I170517 07:32:11.077976 21231 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
=== RUN   TestRemovedReplicaTombstone/#10
I170517 07:32:11.110672 21301 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:11.120299 20575 storage/store.go:2139  removing replica
I170517 07:32:11.121690 20575 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
=== RUN   TestRemovedReplicaTombstone/#11
I170517 07:32:11.158550 21232 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:11.163663 21407 storage/store.go:2139  removing replica
I170517 07:32:11.164991 21407 storage/replica.go:684  removed 10 (2+8) keys in 0ms [clear=0ms commit=0ms]
=== RUN   TestRemovedReplicaTombstone/#12
I170517 07:32:11.179106 21384 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:11.195013 21607 storage/store.go:2139  removing replica
I170517 07:32:11.197338 21607 storage/replica.go:684  removed 10 (2+8) keys in 1ms [clear=0ms commit=1ms]
--- PASS: TestRemovedReplicaTombstone (0.46s)
    --- PASS: TestRemovedReplicaTombstone/#00 (0.06s)
    --- PASS: TestRemovedReplicaTombstone/#01 (0.02s)
    --- PASS: TestRemovedReplicaTombstone/#02 (0.02s)
    --- PASS: TestRemovedReplicaTombstone/#03 (0.02s)
    --- PASS: TestRemovedReplicaTombstone/#04 (0.06s)
    --- PASS: TestRemovedReplicaTombstone/#05 (0.03s)
    --- PASS: TestRemovedReplicaTombstone/#06 (0.02s)
    --- PASS: TestRemovedReplicaTombstone/#07 (0.02s)
    --- PASS: TestRemovedReplicaTombstone/#08 (0.02s)
    --- PASS: TestRemovedReplicaTombstone/#09 (0.04s)
    --- PASS: TestRemovedReplicaTombstone/#10 (0.04s)
    --- PASS: TestRemovedReplicaTombstone/#11 (0.04s)
    --- PASS: TestRemovedReplicaTombstone/#12 (0.04s)
=== RUN   TestCanCampaignIdleReplica
I170517 07:32:11.265023 21573 util/stop/stopper.go:505  quiescing; tasks left:
2      gossip/infostore.go:301
I170517 07:32:11.265248 21573 util/stop/stopper.go:505  quiescing; tasks left:
1      gossip/infostore.go:301
--- PASS: TestCanCampaignIdleReplica (0.05s)
=== RUN   TestSendSnapshotThrottling
--- PASS: TestSendSnapshotThrottling (0.01s)
=== RUN   TestReserveSnapshotThrottling
I170517 07:32:11.298149 21715 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestReserveSnapshotThrottling (0.10s)
=== RUN   TestSnapshotRateLimit
=== RUN   TestSnapshotRateLimit/UNKNOWN
=== RUN   TestSnapshotRateLimit/RECOVERY
=== RUN   TestSnapshotRateLimit/REBALANCE
--- PASS: TestSnapshotRateLimit (0.00s)
    --- PASS: TestSnapshotRateLimit/UNKNOWN (0.00s)
    --- PASS: TestSnapshotRateLimit/RECOVERY (0.00s)
    --- PASS: TestSnapshotRateLimit/REBALANCE (0.00s)
=== RUN   TestStoresAddStore
--- PASS: TestStoresAddStore (0.01s)
=== RUN   TestStoresRemoveStore
--- PASS: TestStoresRemoveStore (0.02s)
=== RUN   TestStoresGetStoreCount
--- PASS: TestStoresGetStoreCount (0.02s)
=== RUN   TestStoresVisitStores
--- PASS: TestStoresVisitStores (0.02s)
=== RUN   TestStoresGetStore
--- PASS: TestStoresGetStore (0.03s)
=== RUN   TestStoresLookupReplica
W170517 07:32:11.503959 21810 storage/stores.go:218  range not contained in one range: ["b","d"), but have [/Min,"c")
--- PASS: TestStoresLookupReplica (0.01s)
=== RUN   TestStoresGossipStorage
I170517 07:32:11.525066 21813 storage/stores.go:296  read 0 node addresses from persistent storage
I170517 07:32:11.533086 21813 storage/stores.go:312  wrote 1 node addresses to persistent storage
I170517 07:32:11.533193 21813 storage/stores.go:296  read 1 node addresses from persistent storage
I170517 07:32:11.533543 21813 storage/stores.go:296  read 1 node addresses from persistent storage
--- PASS: TestStoresGossipStorage (0.03s)
=== RUN   TestStoresGossipStorageReadLatest
I170517 07:32:11.545952 21816 storage/stores.go:312  wrote 1 node addresses to persistent storage
I170517 07:32:11.547009 21816 storage/stores.go:312  wrote 2 node addresses to persistent storage
I170517 07:32:11.547175 21816 storage/stores.go:296  read 2 node addresses from persistent storage
I170517 07:32:11.549822 21816 storage/stores.go:296  read 2 node addresses from persistent storage
--- PASS: TestStoresGossipStorageReadLatest (0.02s)
=== RUN   TestTimedMutex
--- PASS: TestTimedMutex (0.02s)
=== RUN   TestAssertHeld
--- PASS: TestAssertHeld (0.00s)
=== RUN   TestTimestampCache
--- PASS: TestTimestampCache (0.01s)
=== RUN   TestTimestampCacheEviction
--- PASS: TestTimestampCacheEviction (0.02s)
=== RUN   TestTimestampCacheNoEviction
--- PASS: TestTimestampCacheNoEviction (0.02s)
=== RUN   TestTimestampCacheExpandRequests
--- PASS: TestTimestampCacheExpandRequests (0.01s)
=== RUN   TestTimestampCacheLayeredIntervals
=== RUN   TestTimestampCacheLayeredIntervals/#00
=== RUN   TestTimestampCacheLayeredIntervals/#00/simultaneous=false
=== RUN   TestTimestampCacheLayeredIntervals/#00/simultaneous=false/reverse=false
=== RUN   TestTimestampCacheLayeredIntervals/#00/simultaneous=false/reverse=false/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#00/simultaneous=false/reverse=false/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#00/simultaneous=false/reverse=true
=== RUN   TestTimestampCacheLayeredIntervals/#00/simultaneous=false/reverse=true/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#00/simultaneous=false/reverse=true/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#00/simultaneous=true
=== RUN   TestTimestampCacheLayeredIntervals/#00/simultaneous=true/reverse=false
=== RUN   TestTimestampCacheLayeredIntervals/#00/simultaneous=true/reverse=false/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#00/simultaneous=true/reverse=false/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#00/simultaneous=true/reverse=true
=== RUN   TestTimestampCacheLayeredIntervals/#00/simultaneous=true/reverse=true/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#00/simultaneous=true/reverse=true/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#01
=== RUN   TestTimestampCacheLayeredIntervals/#01/simultaneous=false
=== RUN   TestTimestampCacheLayeredIntervals/#01/simultaneous=false/reverse=false
=== RUN   TestTimestampCacheLayeredIntervals/#01/simultaneous=false/reverse=false/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#01/simultaneous=false/reverse=false/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#01/simultaneous=false/reverse=true
=== RUN   TestTimestampCacheLayeredIntervals/#01/simultaneous=false/reverse=true/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#01/simultaneous=false/reverse=true/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#01/simultaneous=true
=== RUN   TestTimestampCacheLayeredIntervals/#01/simultaneous=true/reverse=false
=== RUN   TestTimestampCacheLayeredIntervals/#01/simultaneous=true/reverse=false/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#01/simultaneous=true/reverse=false/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#01/simultaneous=true/reverse=true
=== RUN   TestTimestampCacheLayeredIntervals/#01/simultaneous=true/reverse=true/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#01/simultaneous=true/reverse=true/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#02
=== RUN   TestTimestampCacheLayeredIntervals/#02/simultaneous=false
=== RUN   TestTimestampCacheLayeredIntervals/#02/simultaneous=false/reverse=false
=== RUN   TestTimestampCacheLayeredIntervals/#02/simultaneous=false/reverse=false/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#02/simultaneous=false/reverse=false/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#02/simultaneous=false/reverse=true
=== RUN   TestTimestampCacheLayeredIntervals/#02/simultaneous=false/reverse=true/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#02/simultaneous=false/reverse=true/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#02/simultaneous=true
=== RUN   TestTimestampCacheLayeredIntervals/#02/simultaneous=true/reverse=false
=== RUN   TestTimestampCacheLayeredIntervals/#02/simultaneous=true/reverse=false/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#02/simultaneous=true/reverse=false/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#02/simultaneous=true/reverse=true
=== RUN   TestTimestampCacheLayeredIntervals/#02/simultaneous=true/reverse=true/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#02/simultaneous=true/reverse=true/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#03
=== RUN   TestTimestampCacheLayeredIntervals/#03/simultaneous=false
=== RUN   TestTimestampCacheLayeredIntervals/#03/simultaneous=false/reverse=false
=== RUN   TestTimestampCacheLayeredIntervals/#03/simultaneous=false/reverse=false/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#03/simultaneous=false/reverse=false/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#03/simultaneous=false/reverse=true
=== RUN   TestTimestampCacheLayeredIntervals/#03/simultaneous=false/reverse=true/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#03/simultaneous=false/reverse=true/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#03/simultaneous=true
=== RUN   TestTimestampCacheLayeredIntervals/#03/simultaneous=true/reverse=false
=== RUN   TestTimestampCacheLayeredIntervals/#03/simultaneous=true/reverse=false/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#03/simultaneous=true/reverse=false/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#03/simultaneous=true/reverse=true
=== RUN   TestTimestampCacheLayeredIntervals/#03/simultaneous=true/reverse=true/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#03/simultaneous=true/reverse=true/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#04
=== RUN   TestTimestampCacheLayeredIntervals/#04/simultaneous=false
=== RUN   TestTimestampCacheLayeredIntervals/#04/simultaneous=false/reverse=false
=== RUN   TestTimestampCacheLayeredIntervals/#04/simultaneous=false/reverse=false/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#04/simultaneous=false/reverse=false/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#04/simultaneous=false/reverse=true
=== RUN   TestTimestampCacheLayeredIntervals/#04/simultaneous=false/reverse=true/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#04/simultaneous=false/reverse=true/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#04/simultaneous=true
=== RUN   TestTimestampCacheLayeredIntervals/#04/simultaneous=true/reverse=false
=== RUN   TestTimestampCacheLayeredIntervals/#04/simultaneous=true/reverse=false/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#04/simultaneous=true/reverse=false/sameTxn=true
=== RUN   TestTimestampCacheLayeredIntervals/#04/simultaneous=true/reverse=true
=== RUN   TestTimestampCacheLayeredIntervals/#04/simultaneous=true/reverse=true/sameTxn=false
=== RUN   TestTimestampCacheLayeredIntervals/#04/simultaneous=true/reverse=true/sameTxn=true
--- PASS: TestTimestampCacheLayeredIntervals (0.05s)
    --- PASS: TestTimestampCacheLayeredIntervals/#00 (0.01s)
        --- PASS: TestTimestampCacheLayeredIntervals/#00/simultaneous=false (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#00/simultaneous=false/reverse=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#00/simultaneous=false/reverse=false/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#00/simultaneous=false/reverse=false/sameTxn=true (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#00/simultaneous=false/reverse=true (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#00/simultaneous=false/reverse=true/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#00/simultaneous=false/reverse=true/sameTxn=true (0.00s)
        --- PASS: TestTimestampCacheLayeredIntervals/#00/simultaneous=true (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#00/simultaneous=true/reverse=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#00/simultaneous=true/reverse=false/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#00/simultaneous=true/reverse=false/sameTxn=true (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#00/simultaneous=true/reverse=true (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#00/simultaneous=true/reverse=true/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#00/simultaneous=true/reverse=true/sameTxn=true (0.00s)
    --- PASS: TestTimestampCacheLayeredIntervals/#01 (0.01s)
        --- PASS: TestTimestampCacheLayeredIntervals/#01/simultaneous=false (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#01/simultaneous=false/reverse=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#01/simultaneous=false/reverse=false/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#01/simultaneous=false/reverse=false/sameTxn=true (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#01/simultaneous=false/reverse=true (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#01/simultaneous=false/reverse=true/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#01/simultaneous=false/reverse=true/sameTxn=true (0.00s)
        --- PASS: TestTimestampCacheLayeredIntervals/#01/simultaneous=true (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#01/simultaneous=true/reverse=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#01/simultaneous=true/reverse=false/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#01/simultaneous=true/reverse=false/sameTxn=true (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#01/simultaneous=true/reverse=true (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#01/simultaneous=true/reverse=true/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#01/simultaneous=true/reverse=true/sameTxn=true (0.00s)
    --- PASS: TestTimestampCacheLayeredIntervals/#02 (0.01s)
        --- PASS: TestTimestampCacheLayeredIntervals/#02/simultaneous=false (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#02/simultaneous=false/reverse=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#02/simultaneous=false/reverse=false/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#02/simultaneous=false/reverse=false/sameTxn=true (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#02/simultaneous=false/reverse=true (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#02/simultaneous=false/reverse=true/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#02/simultaneous=false/reverse=true/sameTxn=true (0.00s)
        --- PASS: TestTimestampCacheLayeredIntervals/#02/simultaneous=true (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#02/simultaneous=true/reverse=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#02/simultaneous=true/reverse=false/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#02/simultaneous=true/reverse=false/sameTxn=true (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#02/simultaneous=true/reverse=true (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#02/simultaneous=true/reverse=true/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#02/simultaneous=true/reverse=true/sameTxn=true (0.00s)
    --- PASS: TestTimestampCacheLayeredIntervals/#03 (0.01s)
        --- PASS: TestTimestampCacheLayeredIntervals/#03/simultaneous=false (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#03/simultaneous=false/reverse=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#03/simultaneous=false/reverse=false/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#03/simultaneous=false/reverse=false/sameTxn=true (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#03/simultaneous=false/reverse=true (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#03/simultaneous=false/reverse=true/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#03/simultaneous=false/reverse=true/sameTxn=true (0.00s)
        --- PASS: TestTimestampCacheLayeredIntervals/#03/simultaneous=true (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#03/simultaneous=true/reverse=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#03/simultaneous=true/reverse=false/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#03/simultaneous=true/reverse=false/sameTxn=true (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#03/simultaneous=true/reverse=true (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#03/simultaneous=true/reverse=true/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#03/simultaneous=true/reverse=true/sameTxn=true (0.00s)
    --- PASS: TestTimestampCacheLayeredIntervals/#04 (0.00s)
        --- PASS: TestTimestampCacheLayeredIntervals/#04/simultaneous=false (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#04/simultaneous=false/reverse=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#04/simultaneous=false/reverse=false/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#04/simultaneous=false/reverse=false/sameTxn=true (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#04/simultaneous=false/reverse=true (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#04/simultaneous=false/reverse=true/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#04/simultaneous=false/reverse=true/sameTxn=true (0.00s)
        --- PASS: TestTimestampCacheLayeredIntervals/#04/simultaneous=true (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#04/simultaneous=true/reverse=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#04/simultaneous=true/reverse=false/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#04/simultaneous=true/reverse=false/sameTxn=true (0.00s)
            --- PASS: TestTimestampCacheLayeredIntervals/#04/simultaneous=true/reverse=true (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#04/simultaneous=true/reverse=true/sameTxn=false (0.00s)
                --- PASS: TestTimestampCacheLayeredIntervals/#04/simultaneous=true/reverse=true/sameTxn=true (0.00s)
=== RUN   TestTimestampCacheClear
--- PASS: TestTimestampCacheClear (0.01s)
=== RUN   TestTimestampCacheReadVsWrite
--- PASS: TestTimestampCacheReadVsWrite (0.01s)
=== RUN   TestTimestampCacheEqualTimestamps
--- PASS: TestTimestampCacheEqualTimestamps (0.01s)
=== RUN   TestBelowRaftProtos
--- PASS: TestBelowRaftProtos (0.01s)
=== RUN   TestStoreRangeLease
=== RUN   TestStoreRangeLease/epoch-based_leases?_true
I170517 07:32:11.761936 21836 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:11.762127 21836 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:55482" > attrs:<> locality:<>
I170517 07:32:11.771077 21928 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r2]
E170517 07:32:11.780347 21898 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:32:11.781063 21703 storage/replica_command.go:2633  [s1,r2/1:{a-/Max}] initiating a split of this range at key "b" [r3]
E170517 07:32:11.781201 21898 storage/queue.go:634  [replicate,s1,r2/1:{a-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
W170517 07:32:11.785594 21706 storage/stores.go:218  range not contained in one range: [/Meta2/"b","b\x00"), but have [/Min,"a")
E170517 07:32:11.799667 21898 storage/queue.go:634  [replicate,s1,r2/1:{a-b}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:32:11.805269 21898 storage/queue.go:634  [replicate,s1,r3/1:{b-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:32:11.806364 22050 storage/replica_command.go:2633  [s1,r3/1:{b-/Max}] initiating a split of this range at key "c" [r4]
E170517 07:32:11.819760 21898 storage/queue.go:634  [replicate,s1,r3/1:{b-c}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:32:11.820011 21898 storage/queue.go:634  [replicate,s1,r4/1:{c-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:32:11.822253 21836 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
I170517 07:32:11.825145 21970 storage/replica_proposal.go:385  [s1,r2/1:{a-b}] new range lease repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,7 following repl=(n1,s1):1 start=0.000000000,0 exp=0.900000123,5 pro=0.000000123,6
I170517 07:32:11.832941 21972 storage/replica_proposal.go:385  [s1,r3/1:{b-c}] new range lease repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,12 following repl=(n1,s1):1 start=0.000000000,0 exp=0.900000123,5 pro=0.000000123,6
I170517 07:32:11.836092 21975 storage/replica_proposal.go:385  [s1,r2/1:{a-b}] new range lease repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,17 following repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,7
I170517 07:32:11.837096 21976 storage/replica_proposal.go:385  [s1,r3/1:{b-c}] new range lease repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,39 following repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,12
I170517 07:32:11.845644 21980 storage/replica_proposal.go:385  [s1,r4/1:{c-/Max}] new range lease repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,64 following repl=(n1,s1):1 start=0.000000000,0 exp=0.900000123,5 pro=0.000000123,6
=== RUN   TestStoreRangeLease/epoch-based_leases?_false
I170517 07:32:11.882766 22083 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:11.888631 22083 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:49299" > attrs:<> locality:<>
I170517 07:32:11.895877 22178 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r2]
E170517 07:32:11.906807 21495 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:32:11.907522 21495 storage/queue.go:634  [replicate,s1,r2/1:{a-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:32:11.908487 22170 storage/replica_command.go:2633  [s1,r2/1:{a-/Max}] initiating a split of this range at key "b" [r3]
W170517 07:32:11.911888 22056 storage/stores.go:218  range not contained in one range: [/Meta2/"b","b\x00"), but have [/Min,"a")
E170517 07:32:11.926770 21495 storage/queue.go:634  [replicate,s1,r2/1:{a-b}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:32:11.928680 21495 storage/queue.go:634  [replicate,s1,r3/1:{b-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:32:11.946893 22176 storage/replica_command.go:2633  [s1,r3/1:{b-/Max}] initiating a split of this range at key "c" [r4]
E170517 07:32:11.960599 21495 storage/queue.go:634  [replicate,s1,r3/1:{b-c}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:32:11.961138 21495 storage/queue.go:634  [replicate,s1,r4/1:{c-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:32:11.964985 22083 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
I170517 07:32:11.988678 22091 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:49299->127.0.0.1:46063: use of closed network connection
I170517 07:32:11.988773 22026 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
--- PASS: TestStoreRangeLease (0.29s)
    --- PASS: TestStoreRangeLease/epoch-based_leases?_true (0.12s)
    --- PASS: TestStoreRangeLease/epoch-based_leases?_false (0.14s)
=== RUN   TestStoreRangeLeaseSwitcheroo
I170517 07:32:12.028653 22049 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:12.028896 22049 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:32800" > attrs:<> locality:<>
I170517 07:32:12.040437 22340 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r2]
E170517 07:32:12.049220 22327 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:32:12.049753 22049 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
I170517 07:32:12.052742 22240 storage/replica_proposal.go:385  [s1,r2/1:{a-/Max}] new range lease repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,3 following repl=(n1,s1):1 start=0.000000000,0 exp=0.900000123,5 pro=0.000000123,6
I170517 07:32:12.060005 22240 storage/replica_proposal.go:385  [s1,r2/1:{a-/Max}] new range lease repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,13 following repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,3
E170517 07:32:12.061044 22327 storage/queue.go:634  [replicate,s1,r2/1:{a-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:32:12.075716 22049 storage/client_test.go:1173  test clock advanced to: 3.600000127,0
I170517 07:32:12.083037 22049 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/replica.go:4354
1      storage/replica_range_lease.go:243
1      storage/client_test.go:510
W170517 07:32:12.085380 20922 storage/replica.go:2296  [s1,r1/1:{/Min-a}] shutdown cancellation after 0.0s of attempting command RequestLease [/Min,/Min)
I170517 07:32:12.085580 22049 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/replica.go:4354
1      storage/client_test.go:510
I170517 07:32:12.085736 22049 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/client_test.go:510
W170517 07:32:12.086407 22187 storage/replica.go:2296  [hb,s1,r1/1:{/Min-a}] shutdown cancellation after 0.0s of attempting command [txn: 710c2b14], BeginTransaction [/System/NodeLiveness/1,/Min), ConditionalPut [/System/NodeLiveness/1,/Min), EndTransaction [/System/NodeLiveness/1,/Min)
I170517 07:32:12.088341 22183 storage/node_liveness.go:352  [hb] heartbeat result is ambiguous (server shutdown); retrying
W170517 07:32:12.088833 22183 storage/node_liveness.go:253  [hb] failed node liveness heartbeat: node unavailable; try another peer
I170517 07:32:12.121056 22049 storage/client_test.go:1173  test clock advanced to: 5.400000129,0
I170517 07:32:12.128258 22435 storage/replica_proposal.go:385  [s1,r2/1:{a-/Max}] new range lease repl=(n1,s1):1 start=0.000000000,0 epo=2 pro=3.600000127,78 following repl=(n1,s1):1 start=0.000000000,0 exp=4.500000127,2 pro=3.600000127,3
I170517 07:32:12.133506 22439 storage/replica_proposal.go:385  [s1,r2/1:{a-/Max}] new range lease repl=(n1,s1):1 start=0.000000000,0 epo=3 pro=5.400000129,34 following repl=(n1,s1):1 start=0.000000000,0 epo=2 pro=3.600000127,78
--- PASS: TestStoreRangeLeaseSwitcheroo (0.14s)
=== RUN   TestStoreGossipSystemData
I170517 07:32:12.176104 22351 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:12.176271 22351 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:37537" > attrs:<> locality:<>
I170517 07:32:12.191481 22619 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key /Table/0 [r2]
E170517 07:32:12.201628 22628 storage/queue.go:634  [replicate,s1,r1/1:/{Min-Table/0}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:32:12.202267 22628 storage/queue.go:634  [replicate,s1,r2/1:/{Table/0-Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:32:12.225347 22647 storage/replica_proposal.go:385  [s1,r2/1:/{Table/0-Max}] new range lease repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=0.000000123,104 following repl=(n1,s1):1 start=0.000000000,0 exp=0.900000123,6 pro=0.000000123,7
W170517 07:32:12.229414 22717 storage/node_liveness.go:253  [hb] failed node liveness heartbeat: node unavailable; try another peer
I170517 07:32:12.230364 22250 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:37537->127.0.0.1:46585: use of closed network connection
--- PASS: TestStoreGossipSystemData (0.09s)
=== RUN   TestStoreRangeMergeTwoEmptyRanges
I170517 07:32:12.241624 22730 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:32:12.264346 22730 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "b" [r2]
I170517 07:32:12.275117 22730 storage/replica_command.go:3123  [s1,r1/1:{/Min-b}] initiating a merge of [n1,s1,r2/1:{b-/Max}] into this range
W170517 07:32:12.278699 22730 storage/stores.go:218  range not contained in one range: [/Min,"b\x00"), but have [/Min,"b")
I170517 07:32:12.285923 22744 storage/store.go:2139  [s1,r1/1:{/Min-b}] removing replica
--- PASS: TestStoreRangeMergeTwoEmptyRanges (0.09s)
=== RUN   TestStoreRangeMergeMetadataCleanup
I170517 07:32:12.333821 22850 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:32:12.361634 22850 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "b" [r2]
I170517 07:32:12.371649 22850 storage/replica_command.go:3123  [s1,r1/1:{/Min-b}] initiating a merge of [n1,s1,r2/1:{b-/Max}] into this range
W170517 07:32:12.374736 22850 storage/stores.go:218  range not contained in one range: [/Min,"b\x00"), but have [/Min,"b")
I170517 07:32:12.384581 22869 storage/store.go:2139  [s1,r1/1:{/Min-b}] removing replica
W170517 07:32:12.386073 22639 storage/replica.go:2296  [s1,r1/1:/M{in-ax}] shutdown cancellation after 0.0s of attempting command PushTxn [/Local/Range/""/RangeDescriptor,/Min)
W170517 07:32:12.386235 22639 storage/intent_resolver.go:313  [n1,s1,r1/1:/M{in-ax}]: failed to push during intent resolution: result is ambiguous (server shutdown)
I170517 07:32:12.386335 22850 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
1      kv/txn_coord_sender.go:981
I170517 07:32:12.386441 22850 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
--- PASS: TestStoreRangeMergeMetadataCleanup (0.08s)
=== RUN   TestStoreRangeMergeWithData
I170517 07:32:12.416154 22962 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:32:12.449414 22962 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "b" [r2]
I170517 07:32:12.461211 22962 storage/replica_command.go:3123  [s1,r1/1:{/Min-b}] initiating a merge of [n1,s1,r2/1:{b-/Max}] into this range
W170517 07:32:12.464537 22962 storage/stores.go:218  range not contained in one range: [/Min,"b\x00"), but have [/Min,"b")
I170517 07:32:12.471086 22943 storage/store.go:2139  [s1,r1/1:{/Min-b}] removing replica
I170517 07:32:12.478860 22962 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
W170517 07:32:12.479001 23050 storage/replica.go:2296  [s1,r1/1:/M{in-ax}] shutdown cancellation after 0.0s of attempting command ResolveIntent [/Meta2/Max,/Min)
W170517 07:32:12.479209 23050 storage/intent_resolver.go:309  [n1,s1,r1/1:/M{in-ax}]: failed to resolve intents: result is ambiguous (server shutdown)
--- PASS: TestStoreRangeMergeWithData (0.09s)
=== RUN   TestStoreRangeMergeLastRange
I170517 07:32:12.500024 23061 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
--- PASS: TestStoreRangeMergeLastRange (0.04s)
=== RUN   TestStoreRangeMergeNonCollocated
I170517 07:32:12.569561 22954 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:12.569913 22954 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:33508" > attrs:<> locality:<>
W170517 07:32:12.584802 22954 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:12.586694 23248 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:33508
I170517 07:32:12.596762 22954 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:47407" > attrs:<> locality:<>
W170517 07:32:12.607595 22954 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:12.618797 22830 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:33508
I170517 07:32:12.624201 22954 storage/store.go:1252  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I170517 07:32:12.624401 22954 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:39711" > attrs:<> locality:<>
W170517 07:32:12.640467 22954 gossip/gossip.go:1196  [n4] no incoming or outgoing connections
I170517 07:32:12.641797 23280 gossip/client.go:131  [n4] started gossip client to 127.0.0.1:33508
I170517 07:32:12.653025 22954 gossip/gossip.go:297  [n4] NodeDescriptor set to node_id:4 address:<network_field:"tcp" address_field:"127.0.0.1:48441" > attrs:<> locality:<>
I170517 07:32:12.676871 22954 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "d" [r2]
I170517 07:32:12.692223 22954 storage/replica_command.go:2633  [s1,r1/1:{/Min-d}] initiating a split of this range at key "b" [r3]
E170517 07:32:12.692936 23153 storage/queue.go:634  [replicate,s1,r1/1:{/Min-d}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:32:12.694846 23153 storage/queue.go:634  [replicate,s1,r2/1:{d-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:32:12.701482 23153 storage/queue.go:634  [replicate,s1,r1/1:{/Min-b}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:32:12.702720 23153 storage/queue.go:634  [replicate,s1,r3/1:{b-d}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:32:12.704544 23482 storage/replica_raftstorage.go:416  [s1,r1/1:{/Min-b}] generated preemptive snapshot 9a2be9c8 at index 21
I170517 07:32:12.706844 23482 storage/store.go:3345  [s1,r1/1:{/Min-b}] streamed snapshot to (n2,s2):?: kv pairs: 24, log entries: 11, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:12.707836 23433 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 21 (id=9a2be9c8, encoded size=8186, 1 rocksdb batches, 11 log entries)
I170517 07:32:12.724423 23433 storage/replica_raftstorage.go:605  [s2,r1/?:{/Min-b}] applied preemptive snapshot in 5ms [clear=0ms batch=0ms entries=1ms commit=1ms]
I170517 07:32:12.726580 23482 storage/replica_command.go:3538  [s1,r1/1:{/Min-b}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:{/Min-b} [(n1,s1):1, next=2]
I170517 07:32:12.733851 23664 storage/replica.go:2590  [s1,r1/1:{/Min-b}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:12.749739 23665 storage/replica_raftstorage.go:416  [s1,r1/1:{/Min-b}] generated preemptive snapshot 83aedee2 at index 23
I170517 07:32:12.751917 23665 storage/store.go:3345  [s1,r1/1:{/Min-b}] streamed snapshot to (n3,s3):?: kv pairs: 27, log entries: 13, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:12.754893 23649 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 23 (id=83aedee2, encoded size=9734, 1 rocksdb batches, 13 log entries)
I170517 07:32:12.756875 23649 storage/replica_raftstorage.go:605  [s3,r1/?:{/Min-b}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:12.758733 23665 storage/replica_command.go:3538  [s1,r1/1:{/Min-b}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:{/Min-b} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:12.765448 23674 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:12.773105 23544 storage/replica.go:2590  [s1,r1/1:{/Min-b}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:13.059695 23714 storage/replica_raftstorage.go:416  [s1,r3/1:{b-d}] generated preemptive snapshot 7a6b80b5 at index 11
I170517 07:32:13.061671 23714 storage/store.go:3345  [s1,r3/1:{b-d}] streamed snapshot to (n2,s2):?: kv pairs: 8, log entries: 1, rate-limit: 2.0 MiB/sec, 1ms
I170517 07:32:13.062662 23692 storage/replica_raftstorage.go:597  [s2,r3/?:{-}] applying preemptive snapshot at index 11 (id=7a6b80b5, encoded size=459, 1 rocksdb batches, 1 log entries)
I170517 07:32:13.063469 23692 storage/replica_raftstorage.go:605  [s2,r3/?:{b-d}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:13.065181 23714 storage/replica_command.go:3538  [s1,r3/1:{b-d}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r3:{b-d} [(n1,s1):1, next=2]
I170517 07:32:13.076330 23703 storage/replica.go:2590  [s1,r3/1:{b-d}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:13.080236 23705 storage/replica_raftstorage.go:416  [s1,r3/1:{b-d}] generated preemptive snapshot 73b36888 at index 13
I170517 07:32:13.082288 23705 storage/store.go:3345  [s1,r3/1:{b-d}] streamed snapshot to (n4,s4):?: kv pairs: 10, log entries: 3, rate-limit: 2.0 MiB/sec, 1ms
I170517 07:32:13.083239 23551 storage/replica_raftstorage.go:597  [s4,r3/?:{-}] applying preemptive snapshot at index 13 (id=73b36888, encoded size=1969, 1 rocksdb batches, 3 log entries)
I170517 07:32:13.084296 23551 storage/replica_raftstorage.go:605  [s4,r3/?:{b-d}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:13.090342 23705 storage/replica_command.go:3538  [s1,r3/1:{b-d}] change replicas (ADD_REPLICA (n4,s4):3): read existing descriptor r3:{b-d} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:13.119495 23393 storage/replica.go:2590  [s1,r3/1:{b-d}] proposing ADD_REPLICA (n4,s4):3: [(n1,s1):1 (n2,s2):2 (n4,s4):3]
I170517 07:32:13.163556 23730 storage/replica_raftstorage.go:416  [s1,r2/1:{d-/Max}] generated preemptive snapshot f0ce6081 at index 11
I170517 07:32:13.165427 23730 storage/store.go:3345  [s1,r2/1:{d-/Max}] streamed snapshot to (n2,s2):?: kv pairs: 27, log entries: 1, rate-limit: 2.0 MiB/sec, 1ms
I170517 07:32:13.166430 23748 storage/replica_raftstorage.go:597  [s2,r2/?:{-}] applying preemptive snapshot at index 11 (id=f0ce6081, encoded size=3537, 1 rocksdb batches, 1 log entries)
I170517 07:32:13.167281 23748 storage/replica_raftstorage.go:605  [s2,r2/?:{d-/Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:13.168925 23730 storage/replica_command.go:3538  [s1,r2/1:{d-/Max}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r2:{d-/Max} [(n1,s1):1, next=2]
I170517 07:32:13.185402 23749 storage/replica.go:2590  [s1,r2/1:{d-/Max}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:13.193342 23552 storage/replica_raftstorage.go:416  [s1,r2/1:{d-/Max}] generated preemptive snapshot a33b3006 at index 13
I170517 07:32:13.195933 23552 storage/store.go:3345  [s1,r2/1:{d-/Max}] streamed snapshot to (n3,s3):?: kv pairs: 29, log entries: 3, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:13.196957 23734 storage/replica_raftstorage.go:597  [s3,r2/?:{-}] applying preemptive snapshot at index 13 (id=a33b3006, encoded size=5052, 1 rocksdb batches, 3 log entries)
I170517 07:32:13.197976 23734 storage/replica_raftstorage.go:605  [s3,r2/?:{d-/Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:13.204863 23552 storage/replica_command.go:3538  [s1,r2/1:{d-/Max}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r2:{d-/Max} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:13.239020 23755 storage/replica.go:2590  [s1,r2/1:{d-/Max}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:13.278764 22954 storage/replica_command.go:3123  initiating a merge of [n1,s1,r3/1:{b-d}] into this range
W170517 07:32:13.296280 23688 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n1,s1):?
W170517 07:32:13.296797 23676 storage/store.go:3156  [s2] raft error: node 1 claims to not contain store 1 for replica (n1,s1):?: store 1 was not found
W170517 07:32:13.297174 23674 storage/raft_transport.go:442  raft transport stream to node 1 failed: store 1 was not found
W170517 07:32:13.297606 23688 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n1,s1):1
I170517 07:32:13.299229 23735 storage/raft_transport.go:436  raft transport stream to node 1 established
W170517 07:32:13.300045 23510 storage/raft_transport.go:257  unable to accept Raft message from (n3,s3):3: no handler registered for (n1,s1):1
W170517 07:32:13.300515 23737 storage/store.go:3156  [s3] raft error: node 1 claims to not contain store 1 for replica (n1,s1):1: store 1 was not found
I170517 07:32:13.309269 23278 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:48441->127.0.0.1:52998: use of closed network connection
I170517 07:32:13.310306 23369 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:39711->127.0.0.1:43577: use of closed network connection
I170517 07:32:13.311227 23269 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:47407->127.0.0.1:51571: use of closed network connection
I170517 07:32:13.311302 23148 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:33508->127.0.0.1:53396: use of closed network connection
I170517 07:32:13.311459 22960 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:13.316918 23155 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:33508: getsockopt: connection refused"; Reconnecting to {127.0.0.1:33508 <nil>}
I170517 07:32:13.317143 23155 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestStoreRangeMergeNonCollocated (0.78s)
=== RUN   TestStoreRangeMergeStats
I170517 07:32:13.326937 23713 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:32:13.338746 23713 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:13.347579 23713 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "b" [r2]
I170517 07:32:13.754408 23713 storage/replica_command.go:3123  [s1,r1/1:{/Min-b}] initiating a merge of [n1,s1,r2/1:{b-/Max}] into this range
W170517 07:32:13.757716 23713 storage/stores.go:218  range not contained in one range: [/Min,"b\x00"), but have [/Min,"b")
I170517 07:32:13.785540 23808 storage/store.go:2139  [s1,r1/1:{/Min-b}] removing replica
I170517 07:32:13.786772 23713 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/store.go:1453
1      storage/intent_resolver.go:259
1      kv/txn_coord_sender.go:981
W170517 07:32:13.787279 23877 gossip/infostore.go:303  [n1] node unavailable; try another peer
I170517 07:32:13.787619 23713 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
W170517 07:32:13.789302 23907 storage/replica.go:2296  [s1,r1/1:/M{in-ax}] shutdown cancellation after 0.0s of attempting command PushTxn [/Local/Range/""/RangeDescriptor,/Min)
W170517 07:32:13.789540 23907 storage/intent_resolver.go:313  [n1,s1,r1/1:/M{in-ax}]: failed to push during intent resolution: result is ambiguous (server shutdown)
--- PASS: TestStoreRangeMergeStats (0.48s)
=== RUN   TestStoreMetrics
--- SKIP: TestStoreMetrics (0.01s)
	client_metrics_test.go:169: TODO(mrtracy): #9204
=== RUN   TestRaftLogQueue
I170517 07:32:13.844648 23867 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:13.844897 23867 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:60326" > attrs:<> locality:<>
W170517 07:32:13.863869 23867 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:13.866012 23924 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:60326
I170517 07:32:13.878564 23867 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:43802" > attrs:<> locality:<>
W170517 07:32:13.902066 23867 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:13.911052 23929 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:60326
I170517 07:32:13.927453 23867 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:58355" > attrs:<> locality:<>
I170517 07:32:14.301273 24138 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:14.301362 23775 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:58355->127.0.0.1:43791: use of closed network connection
I170517 07:32:14.301456 23878 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:60326->127.0.0.1:49845: use of closed network connection
I170517 07:32:14.301588 23909 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:14.303278 24019 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:43802->127.0.0.1:41697: use of closed network connection
I170517 07:32:14.303462 24141 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:58355: getsockopt: connection refused"; Reconnecting to {127.0.0.1:58355 <nil>}
I170517 07:32:14.303567 24141 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestRaftLogQueue (0.50s)
=== RUN   TestStoreRecoverFromEngine
I170517 07:32:14.320960 23739 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:32:14.348593 23739 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:14.359777 23739 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "m" [r2]
I170517 07:32:14.393951 23739 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:32:14.419089 23739 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestStoreRecoverFromEngine (0.14s)
=== RUN   TestStoreRecoverWithErrors
I170517 07:32:14.515052 24448 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:32:14.528552 24448 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:14.544951 24448 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:32:14.571537 24448 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestStoreRecoverWithErrors (0.14s)
=== RUN   TestReplicateRange
I170517 07:32:14.645765 24628 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:52906" > attrs:<> locality:<>
W170517 07:32:14.656236 24628 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:14.663867 24628 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:32:14.664061 24628 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:59800" > attrs:<> locality:<>
I170517 07:32:14.669211 24829 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:52906
I170517 07:32:14.743505 24628 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 376af6d8 at index 15
I170517 07:32:14.745962 24628 storage/store.go:3345  streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:14.746637 24254 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=376af6d8, encoded size=5504, 1 rocksdb batches, 5 log entries)
I170517 07:32:14.747828 24254 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:14.751008 24628 storage/replica_command.go:3538  change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:14.755781 24882 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:14.766848 24571 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:52906->127.0.0.1:36994: use of closed network connection
--- PASS: TestReplicateRange (0.18s)
=== RUN   TestRestoreReplicas
I170517 07:32:14.792660 24864 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:14.792892 24864 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:60650" > attrs:<> locality:<>
W170517 07:32:14.795300 24977 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip system config: periodic gossip is disabled
W170517 07:32:14.795533 24978 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip node liveness: periodic gossip is disabled
W170517 07:32:14.816080 24864 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:14.820296 24543 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:60650
I170517 07:32:14.825480 24864 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:32:14.825684 24864 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:40399" > attrs:<> locality:<>
I170517 07:32:14.835801 24864 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot cbad1eb5 at index 15
I170517 07:32:14.838627 24864 storage/store.go:3345  streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:14.839394 25105 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=cbad1eb5, encoded size=5504, 1 rocksdb batches, 5 log entries)
I170517 07:32:14.840541 25105 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:14.842684 24864 storage/replica_command.go:3538  change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:14.846851 25123 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:14.976454 25128 storage/raft_transport.go:436  raft transport stream to node 1 established
W170517 07:32:15.387530 25223 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip node liveness: periodic gossip is disabled
W170517 07:32:15.387676 25222 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip system config: periodic gossip is disabled
W170517 07:32:15.411525 25329 storage/store.go:1339  [s2,r1/2:/M{in-ax}] could not gossip system config: periodic gossip is disabled
W170517 07:32:15.412629 25330 storage/store.go:1339  [s2,r1/2:/M{in-ax}] could not gossip node liveness: periodic gossip is disabled
I170517 07:32:15.858970 24997 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:15.859121 24849 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:40399->127.0.0.1:35425: use of closed network connection
W170517 07:32:15.860148 25125 storage/raft_transport.go:442  raft transport stream to node 2 failed: EOF
W170517 07:32:15.860594 25128 storage/raft_transport.go:442  raft transport stream to node 1 failed: rpc error: code = Internal desc = transport is closing
I170517 07:32:15.860734 25132 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken read tcp 127.0.0.1:35461->127.0.0.1:40399: read: connection reset by peer.
I170517 07:32:15.861561 25000 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestRestoreReplicas (1.10s)
=== RUN   TestFailedReplicaChange
I170517 07:32:15.921987 25350 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:15.922515 25350 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:44409" > attrs:<> locality:<>
W170517 07:32:15.955190 25350 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:15.957188 25110 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:44409
I170517 07:32:16.008843 25350 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:59004" > attrs:<> locality:<>
I170517 07:32:16.030299 25350 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 0c00ffe0 at index 14
I170517 07:32:16.032493 25350 storage/store.go:3345  streamed snapshot to (n2,s2):?: kv pairs: 32, log entries: 4, rate-limit: 2.0 MiB/sec, 1ms
I170517 07:32:16.033165 25575 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 14 (id=0c00ffe0, encoded size=5779, 1 rocksdb batches, 4 log entries)
I170517 07:32:16.034225 25575 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:16.035820 25350 storage/replica_command.go:3538  change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:16.045043 25564 storage/replica_command.go:177  [s1,r1/1:/M{in-ax}] test injecting error: boom
I170517 07:32:16.049839 25350 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot d08c0dce at index 16
I170517 07:32:16.051591 25350 storage/store.go:3345  streamed snapshot to (n2,s2):?: kv pairs: 32, log entries: 6, rate-limit: 2.0 MiB/sec, 1ms
I170517 07:32:16.052065 25120 storage/replica_raftstorage.go:597  [s2,r1/?:/M{in-ax}] applying preemptive snapshot at index 16 (id=d08c0dce, encoded size=7302, 1 rocksdb batches, 6 log entries)
I170517 07:32:16.053333 25120 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:16.057468 25350 storage/replica_command.go:3538  change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:16.062719 25580 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:16.159955 25602 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:16.218286 25468 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:59004->127.0.0.1:38023: use of closed network connection
W170517 07:32:16.219065 25567 storage/raft_transport.go:442  raft transport stream to node 2 failed: EOF
W170517 07:32:16.219217 25602 storage/raft_transport.go:442  raft transport stream to node 1 failed: rpc error: code = Internal desc = transport is closing
--- PASS: TestFailedReplicaChange (0.34s)
=== RUN   TestReplicateAfterTruncation
I170517 07:32:16.257368 25634 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:36810" > attrs:<> locality:<>
W170517 07:32:16.266387 25634 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:16.267920 25731 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:36810
I170517 07:32:16.278948 25634 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:42000" > attrs:<> locality:<>
I170517 07:32:16.290529 25634 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 7b36df72 at index 17
I170517 07:32:16.295847 25844 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 17 (id=7b36df72, encoded size=4696, 1 rocksdb batches, 2 log entries)
I170517 07:32:16.296804 25844 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:16.299390 25634 storage/store.go:3345  streamed snapshot to (n2,s2):?: kv pairs: 34, log entries: 2, rate-limit: 2.0 MiB/sec, 8ms
I170517 07:32:16.300580 25634 storage/replica_command.go:3538  change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:16.304938 25840 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:16.376132 25874 storage/raft_transport.go:436  raft transport stream to node 1 established
W170517 07:32:16.398866 25847 storage/raft_transport.go:442  raft transport stream to node 2 failed: EOF
--- PASS: TestReplicateAfterTruncation (0.18s)
=== RUN   TestRaftLogSizeAfterTruncation
I170517 07:32:16.436529 25748 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:16.436776 25748 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:48976" > attrs:<> locality:<>
W170517 07:32:16.468842 25748 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:16.482458 25967 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:48976
I170517 07:32:16.490189 25748 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:44796" > attrs:<> locality:<>
W170517 07:32:16.505804 25748 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:16.508277 26116 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:48976
I170517 07:32:16.518813 25748 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:34598" > attrs:<> locality:<>
I170517 07:32:16.539447 26085 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot bae04f15 at index 15
I170517 07:32:16.543897 26202 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=bae04f15, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:16.545069 26202 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:16.545503 26085 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:16.546838 26085 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:16.551235 25148 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:16.556573 26153 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 78d59278 at index 17
I170517 07:32:16.558860 26153 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:16.559836 26156 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=78d59278, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:32:16.562879 26156 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 3ms [clear=0ms batch=0ms entries=2ms commit=0ms]
I170517 07:32:16.564532 26153 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:16.568595 26159 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:16.576524 26259 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:16.893653 25969 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:34598->127.0.0.1:46508: use of closed network connection
I170517 07:32:16.894700 25519 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:48976->127.0.0.1:47099: use of closed network connection
W170517 07:32:16.894899 26239 storage/raft_transport.go:442  raft transport stream to node 3 failed: rpc error: code = Internal desc = transport is closing
I170517 07:32:16.895903 25965 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:44796->127.0.0.1:53582: use of closed network connection
--- PASS: TestRaftLogSizeAfterTruncation (0.49s)
=== RUN   TestSnapshotAfterTruncation
=== RUN   TestSnapshotAfterTruncation/sameTerm
I170517 07:32:16.925599 26121 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:16.925818 26121 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:60464" > attrs:<> locality:<>
W170517 07:32:16.951063 26121 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:16.952798 26418 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:60464
I170517 07:32:16.975875 26121 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:54530" > attrs:<> locality:<>
W170517 07:32:16.986671 26121 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:16.990050 26094 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:60464
I170517 07:32:16.999825 26121 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:33499" > attrs:<> locality:<>
I170517 07:32:17.014680 26431 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 523b83f0 at index 16
I170517 07:32:17.016551 26431 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 34, log entries: 6, rate-limit: 2.0 MiB/sec, 1ms
I170517 07:32:17.017302 26378 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 16 (id=523b83f0, encoded size=5989, 1 rocksdb batches, 6 log entries)
I170517 07:32:17.018484 26378 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:17.020484 26431 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:17.024691 26510 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:17.037290 26385 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 1f309fd7 at index 18
I170517 07:32:17.039098 26385 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 37, log entries: 8, rate-limit: 2.0 MiB/sec, 1ms
I170517 07:32:17.040190 26554 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 18 (id=1f309fd7, encoded size=7536, 1 rocksdb batches, 8 log entries)
I170517 07:32:17.041559 26554 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:17.049933 26385 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:17.055003 26645 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:17.063946 26651 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
W170517 07:32:17.217382 26644 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:17.217836 26674 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):2: store 2 was not found
W170517 07:32:17.218017 26512 storage/raft_transport.go:442  raft transport stream to node 2 failed: store 2 was not found
I170517 07:32:17.226040 26666 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:17.230896 26675 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:17.231770 26691 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):2: store 2 was not found
I170517 07:32:17.233953 26667 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:17.234777 26706 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:17.235218 26656 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):2: store 2 was not found
I170517 07:32:17.236026 26668 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:17.236699 26709 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:17.237141 26708 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):2: store 2 was not found
I170517 07:32:17.257545 26495 storage/raft_transport.go:436  raft transport stream to node 2 established
I170517 07:32:17.261872 26254 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot b97f2250 at index 23
I170517 07:32:17.264374 26254 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):2: kv pairs: 42, log entries: 2, rate-limit: 8.0 MiB/sec, 2ms
I170517 07:32:17.264583 26678 storage/replica_raftstorage.go:597  [s2,r1/2:/M{in-ax}] applying Raft snapshot at index 23 (id=b97f2250, encoded size=5298, 1 rocksdb batches, 2 log entries)
I170517 07:32:17.265642 26678 storage/replica_raftstorage.go:605  [s2,r1/2:/M{in-ax}] applied Raft snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:17.278127 26397 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:54530->127.0.0.1:59660: use of closed network connection
=== RUN   TestSnapshotAfterTruncation/differentTerm
I170517 07:32:17.301398 26640 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:17.302437 26640 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:33482" > attrs:<> locality:<>
W170517 07:32:17.322218 26640 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:17.328901 26933 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:33482
I170517 07:32:17.331380 26640 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:32:17.331582 26640 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:53296" > attrs:<> locality:<>
W170517 07:32:17.345206 26640 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:17.362570 27106 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:33482
I170517 07:32:17.367025 26640 storage/store.go:1252  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I170517 07:32:17.368175 26640 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:57764" > attrs:<> locality:<>
I170517 07:32:17.409135 26927 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 626d8c20 at index 16
I170517 07:32:17.412195 26927 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 34, log entries: 6, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:17.416824 27157 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 16 (id=626d8c20, encoded size=5989, 1 rocksdb batches, 6 log entries)
I170517 07:32:17.419046 27157 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:17.421083 26927 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:17.427912 27024 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:17.432984 27190 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 451c6ace at index 18
I170517 07:32:17.435038 27190 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 37, log entries: 8, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:17.435749 27202 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 18 (id=451c6ace, encoded size=7536, 1 rocksdb batches, 8 log entries)
I170517 07:32:17.438495 27202 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 3ms [clear=0ms batch=0ms entries=2ms commit=0ms]
I170517 07:32:17.444520 27190 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:17.448918 27116 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:17.455290 27120 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
W170517 07:32:17.610970 27115 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:17.611504 27207 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):2: store 2 was not found
W170517 07:32:17.611676 27205 storage/raft_transport.go:442  raft transport stream to node 2 failed: store 2 was not found
I170517 07:32:17.613576 27158 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:17.614380 27162 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:17.614799 27160 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):2: store 2 was not found
I170517 07:32:17.617675 27219 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:17.618481 27195 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:17.618953 27221 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):2: store 2 was not found
I170517 07:32:17.619776 27163 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:17.622140 27192 storage/raft_transport.go:257  unable to accept Raft message from (n3,s3):3: no handler registered for (n1,s1):1
W170517 07:32:17.622732 27118 storage/store.go:3156  [s3] raft error: node 1 claims to not contain store 1 for replica (n1,s1):1: store 1 was not found
W170517 07:32:17.622897 27116 storage/raft_transport.go:442  raft transport stream to node 1 failed: store 1 was not found
W170517 07:32:17.630273 27032 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:17.632830 27197 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):2: store 2 was not found
I170517 07:32:17.937997 27357 storage/raft_transport.go:436  raft transport stream to node 2 established
I170517 07:32:17.939906 27523 storage/raft_transport.go:436  raft transport stream to node 1 established
E170517 07:32:17.944114 27254 storage/replica.go:3386  [s1,r1/1:/M{in-ax}] unable to add replica to Raft repair queue: queue disabled
I170517 07:32:17.976412 26640 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 0ad02963 at index 27
I170517 07:32:17.980457 27361 storage/replica_raftstorage.go:597  [s2,r1/2:/M{in-ax}] applying Raft snapshot at index 27 (id=0ad02963, encoded size=6771, 1 rocksdb batches, 6 log entries)
I170517 07:32:17.981784 27361 storage/replica_raftstorage.go:605  [s2,r1/2:/M{in-ax}] applied Raft snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:17.982649 26640 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):2: kv pairs: 43, log entries: 6, rate-limit: 8.0 MiB/sec, 5ms
W170517 07:32:17.992527 27527 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n1,s1):1
W170517 07:32:17.993998 27525 storage/store.go:3156  [s2] raft error: node 1 claims to not contain store 1 for replica (n1,s1):1: store 1 was not found
I170517 07:32:18.024750 27056 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:57764->127.0.0.1:40079: use of closed network connection
I170517 07:32:18.024829 26798 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:18.025096 26714 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:33482->127.0.0.1:49975: use of closed network connection
I170517 07:32:18.025531 26824 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:18.025920 26803 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:53296->127.0.0.1:54120: use of closed network connection
I170517 07:32:18.026492 26801 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:57764: getsockopt: connection refused"; Reconnecting to {127.0.0.1:57764 <nil>}
I170517 07:32:18.027204 26801 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: context canceled
I170517 07:32:18.028879 26827 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:33482: operation was canceled"; Reconnecting to {127.0.0.1:33482 <nil>}
I170517 07:32:18.028975 26827 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestSnapshotAfterTruncation (1.13s)
    --- PASS: TestSnapshotAfterTruncation/sameTerm (0.37s)
    --- PASS: TestSnapshotAfterTruncation/differentTerm (0.75s)
=== RUN   TestFailedSnapshotFillsReservation
I170517 07:32:18.061618 26942 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:18.061853 26942 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:59295" > attrs:<> locality:<>
W170517 07:32:18.079117 26942 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:18.080319 27541 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:59295
I170517 07:32:18.091100 26942 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:51156" > attrs:<> locality:<>
W170517 07:32:18.108832 26942 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:18.111128 27534 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:59295
I170517 07:32:18.120359 26942 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:43468" > attrs:<> locality:<>
I170517 07:32:18.151667 27642 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:18.161591 27654 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:51156->127.0.0.1:53536: use of closed network connection
--- PASS: TestFailedSnapshotFillsReservation (0.16s)
=== RUN   TestConcurrentRaftSnapshots
I170517 07:32:18.206774 27519 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:18.207006 27519 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:48143" > attrs:<> locality:<>
W170517 07:32:18.230557 27519 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:18.231935 27996 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:48143
I170517 07:32:18.240649 27519 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:36914" > attrs:<> locality:<>
W170517 07:32:18.253805 27519 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:18.258556 28129 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:48143
I170517 07:32:18.271717 27519 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:39887" > attrs:<> locality:<>
W170517 07:32:18.290137 27519 gossip/gossip.go:1196  [n4] no incoming or outgoing connections
I170517 07:32:18.297566 27919 gossip/client.go:131  [n4] started gossip client to 127.0.0.1:48143
I170517 07:32:18.319435 27519 gossip/gossip.go:297  [n4] NodeDescriptor set to node_id:4 address:<network_field:"tcp" address_field:"127.0.0.1:59697" > attrs:<> locality:<>
W170517 07:32:18.332323 27519 gossip/gossip.go:1196  [n5] no incoming or outgoing connections
I170517 07:32:18.333525 28419 gossip/client.go:131  [n5] started gossip client to 127.0.0.1:48143
I170517 07:32:18.335381 28389 gossip/server.go:285  [n1] refusing gossip from node 5 (max 3 conns); forwarding to 3 ({tcp 127.0.0.1:39887})
I170517 07:32:18.343413 28419 gossip/client.go:136  [n5] closing client to node 1 (127.0.0.1:48143): received forward from node 1 to 3 (127.0.0.1:39887)
I170517 07:32:18.344206 28411 gossip/gossip.go:1210  [n5] node has connected to cluster via gossip
I170517 07:32:18.353638 28477 gossip/client.go:131  [n5] started gossip client to 127.0.0.1:39887
I170517 07:32:18.373999 27519 gossip/gossip.go:297  [n5] NodeDescriptor set to node_id:5 address:<network_field:"tcp" address_field:"127.0.0.1:36722" > attrs:<> locality:<>
I170517 07:32:18.386800 28558 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 9d8f84e2 at index 18
I170517 07:32:18.389173 28558 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 36, log entries: 8, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:18.398083 28279 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 18 (id=9d8f84e2, encoded size=6963, 1 rocksdb batches, 8 log entries)
I170517 07:32:18.400830 28279 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 3ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:18.402660 28558 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:18.406942 28511 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:18.412546 28521 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 3799fa29 at index 20
I170517 07:32:18.415272 28521 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 39, log entries: 10, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:18.416519 28393 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 20 (id=3799fa29, encoded size=8523, 1 rocksdb batches, 10 log entries)
I170517 07:32:18.418045 28393 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:18.419860 28521 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:18.424481 28286 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:18.440785 28526 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:18.470130 28596 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot b803023c at index 22
I170517 07:32:18.478440 28596 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n4,s4):?: kv pairs: 42, log entries: 12, rate-limit: 2.0 MiB/sec, 7ms
I170517 07:32:18.479226 28399 storage/replica_raftstorage.go:597  [s4,r1/?:{-}] applying preemptive snapshot at index 22 (id=b803023c, encoded size=10147, 1 rocksdb batches, 12 log entries)
I170517 07:32:18.480853 28399 storage/replica_raftstorage.go:605  [s4,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:18.482667 28596 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n4,s4):4): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:32:18.498257 28587 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n4,s4):4: [(n1,s1):1 (n2,s2):2 (n3,s3):3 (n4,s4):4]
I170517 07:32:18.524081 28590 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 4db3d8c1 at index 24
I170517 07:32:18.530485 28590 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n5,s5):?: kv pairs: 45, log entries: 14, rate-limit: 2.0 MiB/sec, 4ms
I170517 07:32:18.531586 28421 storage/replica_raftstorage.go:597  [s5,r1/?:{-}] applying preemptive snapshot at index 24 (id=4db3d8c1, encoded size=11835, 1 rocksdb batches, 14 log entries)
I170517 07:32:18.539199 28421 storage/replica_raftstorage.go:605  [s5,r1/?:/M{in-ax}] applied preemptive snapshot in 7ms [clear=0ms batch=0ms entries=6ms commit=0ms]
I170517 07:32:18.541554 28590 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n5,s5):5): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, (n4,s4):4, next=5]
I170517 07:32:18.554138 28425 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n5,s5):5: [(n1,s1):1 (n2,s2):2 (n3,s3):3 (n4,s4):4 (n5,s5):5]
W170517 07:32:18.655059 28585 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:18.655928 28602 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
W170517 07:32:18.657378 28600 storage/raft_transport.go:442  raft transport stream to node 3 failed: store 3 was not found
W170517 07:32:18.658396 28585 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
I170517 07:32:18.662296 28592 storage/raft_transport.go:436  raft transport stream to node 3 established
W170517 07:32:18.663652 28637 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:18.664135 28658 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
I170517 07:32:18.665121 28659 storage/raft_transport.go:436  raft transport stream to node 3 established
W170517 07:32:18.666409 28640 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:18.667014 28285 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:18.667329 28285 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:18.667511 28285 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:18.667666 28285 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:18.667962 28639 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
W170517 07:32:18.668059 28578 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):2: store 2 was not found
W170517 07:32:18.669356 28384 storage/raft_transport.go:442  raft transport stream to node 2 failed: store 2 was not found
I170517 07:32:18.718081 28683 storage/raft_transport.go:436  raft transport stream to node 3 established
I170517 07:32:18.719042 28684 storage/raft_transport.go:436  raft transport stream to node 2 established
I170517 07:32:18.750182 28002 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 8475f3f5 at index 28
I170517 07:32:18.756828 28002 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):2: kv pairs: 49, log entries: 1, rate-limit: 8.0 MiB/sec, 5ms
I170517 07:32:18.757309 28787 storage/replica_raftstorage.go:597  [s2,r1/2:/M{in-ax}] applying Raft snapshot at index 28 (id=8475f3f5, encoded size=5459, 1 rocksdb batches, 1 log entries)
I170517 07:32:18.758408 28787 storage/replica_raftstorage.go:605  [s2,r1/2:/M{in-ax}] applied Raft snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:18.762075 28002 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 9a812c88 at index 30
I170517 07:32:18.767088 28002 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):3: kv pairs: 51, log entries: 3, rate-limit: 8.0 MiB/sec, 3ms
I170517 07:32:18.767510 28898 storage/replica_raftstorage.go:597  [s3,r1/3:/M{in-ax}] applying Raft snapshot at index 30 (id=9a812c88, encoded size=6435, 1 rocksdb batches, 3 log entries)
I170517 07:32:18.768657 28898 storage/replica_raftstorage.go:605  [s3,r1/3:/M{in-ax}] applied Raft snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:18.818058 28033 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:59697->127.0.0.1:36013: use of closed network connection
I170517 07:32:18.819978 27994 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:36914->127.0.0.1:49942: use of closed network connection
W170517 07:32:18.820535 28644 storage/raft_transport.go:442  raft transport stream to node 5 failed: rpc error: code = Internal desc = transport is closing
I170517 07:32:18.822349 27882 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:48143->127.0.0.1:48600: use of closed network connection
I170517 07:32:18.823278 28193 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:36722->127.0.0.1:53301: use of closed network connection
--- PASS: TestConcurrentRaftSnapshots (0.64s)
=== RUN   TestReplicateAfterRemoveAndSplit
I170517 07:32:18.862982 28869 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:18.863241 28869 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:50531" > attrs:<> locality:<>
W170517 07:32:18.882450 28869 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:18.885419 29000 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:50531
I170517 07:32:18.895856 28869 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:47117" > attrs:<> locality:<>
W170517 07:32:18.913773 28869 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:18.916623 29016 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:50531
I170517 07:32:18.922503 28869 storage/store.go:1252  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I170517 07:32:18.922739 28869 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:51170" > attrs:<> locality:<>
I170517 07:32:18.935325 29132 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 5e5aac3c at index 15
I170517 07:32:18.937169 29132 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 1ms
I170517 07:32:18.941344 28879 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=5e5aac3c, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:18.942506 28879 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:18.944570 29132 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:18.950415 28881 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:18.955035 29240 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 6dc0d934 at index 17
I170517 07:32:18.958060 29240 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:18.958810 29242 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=6dc0d934, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:32:18.960098 29242 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:18.961745 29240 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:18.965968 29283 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:18.974551 29244 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:19.256404 29231 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
W170517 07:32:19.264553 29290 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:19.265072 29288 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
W170517 07:32:19.265302 29286 storage/raft_transport.go:442  raft transport stream to node 3 failed: store 3 was not found
I170517 07:32:19.266415 29298 storage/raft_transport.go:436  raft transport stream to node 3 established
W170517 07:32:19.267319 29315 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:19.267814 29041 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
I170517 07:32:19.269299 29255 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2]
I170517 07:32:19.270547 29270 storage/raft_transport.go:436  raft transport stream to node 3 established
W170517 07:32:19.274159 28893 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:19.274510 28893 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:19.275381 29272 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
I170517 07:32:19.280301 28869 storage/replica_command.go:2633  initiating a split of this range at key "m" [r2]
E170517 07:32:19.292969 29010 storage/queue.go:634  [replicate,s1,r1/1:{/Min-m}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:32:19.293263 28869 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
I170517 07:32:19.324380 28971 storage/replica_proposal.go:385  [s1,r2/1:{m-/Max}] new range lease repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,19 following repl=(n1,s1):1 start=0.000000000,0 exp=0.900000123,5 pro=0.000000123,8
E170517 07:32:19.334075 28885 storage/queue.go:634  [replicate,s2,r1/2:{/Min-m}] range requires a replication change, but lacks a quorum of live replicas (0/2)
E170517 07:32:19.352549 29431 storage/replica_range_lease.go:197  [s2,r2/2:{m-/Max}] mismatch incrementing epoch for node_id:1 epoch:1 expiration:<wall_time:900000124 logical:25 > ; actual is {NodeID:1 Epoch:1 Expiration:2.700000126,92 Draining:false}
I170517 07:32:19.353720 28983 storage/replica_proposal.go:385  [s1,r2/1:{m-/Max}] new range lease repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,91 following repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,19
E170517 07:32:19.356883 29010 storage/queue.go:634  [replicate,s1,r2/1:{m-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:32:19.385766 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 045ae5f9 at index 14
I170517 07:32:19.389618 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 7fc6747d at index 14
I170517 07:32:19.392661 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 21ded04c at index 14
I170517 07:32:19.395837 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 765bf3c5 at index 14
I170517 07:32:19.405617 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 45776f68 at index 14
I170517 07:32:19.419486 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 9b9b5ae3 at index 14
I170517 07:32:19.422529 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 295f5b6e at index 14
I170517 07:32:19.426282 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 9dcbbd3a at index 14
I170517 07:32:19.433189 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 93f27280 at index 14
I170517 07:32:19.436118 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 7aa44547 at index 14
I170517 07:32:19.439819 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 07c1bdfb at index 14
I170517 07:32:19.454390 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 5798e92b at index 14
I170517 07:32:19.458698 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot af4f6492 at index 14
I170517 07:32:19.462282 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 9f2901ca at index 14
I170517 07:32:19.468825 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 2d99fd5b at index 14
I170517 07:32:19.483784 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 2486e535 at index 14
I170517 07:32:19.488812 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 285481f8 at index 14
I170517 07:32:19.493226 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot b80f923c at index 14
I170517 07:32:19.499468 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 328bc5bf at index 14
I170517 07:32:19.503303 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot fe90cd2e at index 14
I170517 07:32:19.508902 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 1d888fb2 at index 14
I170517 07:32:19.512958 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 355e31c8 at index 14
I170517 07:32:19.516950 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot fd3c0f9b at index 14
I170517 07:32:19.523283 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 01e186f2 at index 14
I170517 07:32:19.531137 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot c277075d at index 14
I170517 07:32:19.543574 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 5fa0fbf0 at index 14
I170517 07:32:19.563503 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 65749727 at index 14
I170517 07:32:19.600685 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot ff700047 at index 14
I170517 07:32:19.670937 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot f0fc4821 at index 14
I170517 07:32:19.811365 29282 storage/store.go:3152  [s3,r1/3:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:19.812906 29227 storage/store.go:3152  [s3,r1/3:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:19.813465 29426 storage/store.go:2139  [replicaGC,s3,r1/3:/M{in-ax}] removing replica
I170517 07:32:19.814302 29426 storage/replica.go:684  [replicaGC,s3,r1/3:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=0ms commit=0ms]
W170517 07:32:19.814761 29417 storage/replica.go:4351  [s3,r1/3:/M{in-ax}] could not acquire lease for range gossip: r1 was not found
W170517 07:32:19.814865 29417 storage/store.go:1339  [s3,r1/3:/M{in-ax}] could not gossip system config: r1 was not found
W170517 07:32:19.815116 29418 storage/replica.go:4351  [s3,r1/3:/M{in-ax}] could not acquire lease for range gossip: r1 was not found
W170517 07:32:19.815208 29418 storage/store.go:1339  [s3,r1/3:/M{in-ax}] could not gossip node liveness: r1 was not found
W170517 07:32:19.816514 29416 storage/replica.go:4351  [s3,r1/3:/M{in-ax}] could not acquire lease for range gossip: r1 was not found
W170517 07:32:19.816937 29416 storage/store.go:1339  [s3,r1/3:/M{in-ax}] could not gossip first range descriptor: r1 was not found
I170517 07:32:19.830684 28869 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot c10f19fa at index 14
I170517 07:32:19.832533 28869 storage/store.go:3345  streamed snapshot to (n3,s3):?: kv pairs: 27, log entries: 4, rate-limit: 2.0 MiB/sec, 1ms
I170517 07:32:19.833548 29424 storage/replica_raftstorage.go:597  [s3,r2/?:{-}] applying preemptive snapshot at index 14 (id=c10f19fa, encoded size=4982, 1 rocksdb batches, 4 log entries)
I170517 07:32:19.834629 29424 storage/replica_raftstorage.go:605  [s3,r2/?:{m-/Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:19.836447 28869 storage/replica_command.go:3538  change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r2:{m-/Max} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:19.846107 29564 storage/replica.go:2590  [s1,r2/1:{m-/Max}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:19.850253 29581 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
I170517 07:32:19.850406 29582 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/client_test.go:510
W170517 07:32:19.850585 29534 storage/replica.go:2296  [s2,r1/2:{/Min-m}] shutdown cancellation after 0.0s of attempting command ResolveIntent [/Meta2/Max,/Min)
W170517 07:32:19.850878 29533 storage/intent_resolver.go:327  [n1,s1,r2/1:{m-/Max}]: failed to resolve intents: result is ambiguous (server shutdown)
W170517 07:32:19.860853 29285 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n1,s1):1
W170517 07:32:19.861215 29285 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n1,s1):1
W170517 07:32:19.863259 29227 storage/raft_transport.go:476  no handler found for store 2 in response range_id:1 from_replica:<node_id:1 store_id:1 replica_id:1 > to_replica:<node_id:2 store_id:2 replica_id:2 > union:<error:<message:"store 1 was not found" transaction_restart:NONE origin_node:0 detail:<store_not_found:<store_id:1 > > now:<wall_time:0 logical:0 > > >
W170517 07:32:19.863647 29227 storage/raft_transport.go:476  no handler found for store 2 in response range_id:2 from_replica:<node_id:1 store_id:1 replica_id:1 > to_replica:<node_id:2 store_id:2 replica_id:2 > union:<error:<message:"store 1 was not found" transaction_restart:NONE origin_node:0 detail:<store_not_found:<store_id:1 > > now:<wall_time:0 logical:0 > > >
I170517 07:32:19.864690 29138 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:51170->127.0.0.1:46042: use of closed network connection
I170517 07:32:19.865000 28917 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:50531->127.0.0.1:55120: use of closed network connection
I170517 07:32:19.865103 28919 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:19.865489 28795 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:47117->127.0.0.1:46848: use of closed network connection
I170517 07:32:19.865717 29032 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:19.866121 28906 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:19.866497 29035 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:51170: getsockopt: connection refused"; Reconnecting to {127.0.0.1:51170 <nil>}
I170517 07:32:19.867180 28922 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:50531: getsockopt: connection refused"; Reconnecting to {127.0.0.1:50531 <nil>}
I170517 07:32:19.868891 28922 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: context canceled
I170517 07:32:19.869128 29035 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: context canceled
--- PASS: TestReplicateAfterRemoveAndSplit (1.05s)
=== RUN   TestRefreshPendingCommands
=== RUN   TestRefreshPendingCommands/#00
I170517 07:32:19.917365 29536 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:19.917771 29536 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:39460" > attrs:<> locality:<>
W170517 07:32:19.922020 29686 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip system config: periodic gossip is disabled
W170517 07:32:19.922555 29687 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip node liveness: periodic gossip is disabled
W170517 07:32:19.972350 29536 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:19.973539 29502 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:39460
I170517 07:32:19.982980 29536 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:53789" > attrs:<> locality:<>
W170517 07:32:19.993992 29536 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:19.998853 29843 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:39460
I170517 07:32:20.009600 29536 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:47486" > attrs:<> locality:<>
I170517 07:32:20.030799 29934 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 5f031803 at index 15
I170517 07:32:20.035146 29934 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 4ms
I170517 07:32:20.037643 29957 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=5f031803, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:20.039440 29957 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:20.044324 29934 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:20.053955 29972 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:20.062462 29598 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 4698b668 at index 17
I170517 07:32:20.066635 29598 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 4ms
I170517 07:32:20.067463 29600 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=4698b668, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:32:20.068815 29600 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:20.088528 29598 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:20.108107 29962 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:20.115961 29978 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
W170517 07:32:20.297706 29979 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:20.298293 29987 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
W170517 07:32:20.298796 29601 storage/raft_transport.go:442  raft transport stream to node 3 failed: store 3 was not found
I170517 07:32:20.303600 29854 storage/raft_transport.go:436  raft transport stream to node 3 established
W170517 07:32:20.305006 29981 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:20.305766 29981 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:20.305903 29856 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
W170517 07:32:20.308211 29981 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
I170517 07:32:20.323881 29536 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
W170517 07:32:20.324780 30092 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip system config: periodic gossip is disabled
W170517 07:32:20.325459 30093 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip node liveness: periodic gossip is disabled
I170517 07:32:20.326233 29536 storage/client_test.go:1173  test clock advanced to: 3.600000127,0
W170517 07:32:20.354559 30188 storage/store.go:1339  [s3,r1/3:/M{in-ax}] could not gossip node liveness: periodic gossip is disabled
W170517 07:32:20.355505 30187 storage/store.go:1339  [s3,r1/3:/M{in-ax}] could not gossip system config: periodic gossip is disabled
I170517 07:32:20.583847 30191 storage/raft_transport.go:436  raft transport stream to node 3 established
I170517 07:32:20.588570 29737 storage/replica_raftstorage.go:416  [raftsnapshot,s2,r1/2:/M{in-ax}] generated Raft snapshot 96f6f6c4 at index 23
I170517 07:32:20.590578 29737 storage/store.go:3345  [raftsnapshot,s2,r1/2:/M{in-ax}] streamed snapshot to (n3,s3):3: kv pairs: 41, log entries: 2, rate-limit: 8.0 MiB/sec, 2ms
I170517 07:32:20.596501 29711 storage/replica_raftstorage.go:597  [s3,r1/3:/M{in-ax}] applying Raft snapshot at index 23 (id=96f6f6c4, encoded size=4816, 1 rocksdb batches, 2 log entries)
I170517 07:32:20.597768 29711 storage/replica_raftstorage.go:605  [s3,r1/3:/M{in-ax}] applied Raft snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
=== RUN   TestRefreshPendingCommands/#01
I170517 07:32:20.694695 30005 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
W170517 07:32:20.694784 30325 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip system config: periodic gossip is disabled
W170517 07:32:20.694850 30326 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip node liveness: periodic gossip is disabled
I170517 07:32:20.694948 30005 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:44833" > attrs:<> locality:<>
W170517 07:32:20.709379 30005 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:20.711741 30203 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:44833
I170517 07:32:20.720121 30005 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:52527" > attrs:<> locality:<>
W170517 07:32:20.739952 30005 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:20.745348 30374 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:44833
I170517 07:32:20.754116 30005 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:38397" > attrs:<> locality:<>
I170517 07:32:20.765401 30548 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 49599a4b at index 15
I170517 07:32:20.767304 30548 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 1ms
I170517 07:32:20.768774 30549 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=49599a4b, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:20.770200 30549 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:20.772110 30548 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:20.778694 30209 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:20.787620 30599 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 5ac6a0ea at index 17
I170517 07:32:20.791015 30599 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:20.791957 30601 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=5ac6a0ea, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:32:20.793242 30601 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:20.796461 30599 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:20.799674 30565 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:20.805334 30556 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
W170517 07:32:20.967058 30627 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:20.967351 30627 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:20.967500 30627 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:20.967950 30617 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
W170517 07:32:20.968097 30615 storage/raft_transport.go:442  raft transport stream to node 3 failed: store 3 was not found
I170517 07:32:20.972926 30618 storage/raft_transport.go:436  raft transport stream to node 3 established
W170517 07:32:20.973888 30558 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:20.974412 30587 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
W170517 07:32:20.996911 30709 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip system config: periodic gossip is disabled
W170517 07:32:20.997094 30710 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip node liveness: periodic gossip is disabled
I170517 07:32:21.007188 30005 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
I170517 07:32:21.009030 30005 storage/client_test.go:1173  test clock advanced to: 3.600000127,0
W170517 07:32:21.021418 30806 storage/store.go:1339  [s3,r1/3:/M{in-ax}] could not gossip system config: periodic gossip is disabled
W170517 07:32:21.023865 30807 storage/store.go:1339  [s3,r1/3:/M{in-ax}] could not gossip node liveness: periodic gossip is disabled
I170517 07:32:21.337996 30560 storage/raft_transport.go:436  raft transport stream to node 3 established
I170517 07:32:21.345001 30454 storage/replica_raftstorage.go:416  [raftsnapshot,s2,r1/2:/M{in-ax}] generated Raft snapshot 6d7f2e70 at index 22
I170517 07:32:21.349619 30454 storage/store.go:3345  [raftsnapshot,s2,r1/2:/M{in-ax}] streamed snapshot to (n3,s3):3: kv pairs: 41, log entries: 1, rate-limit: 8.0 MiB/sec, 4ms
I170517 07:32:21.349805 30728 storage/replica_raftstorage.go:597  [s3,r1/3:/M{in-ax}] applying Raft snapshot at index 22 (id=6d7f2e70, encoded size=4810, 1 rocksdb batches, 1 log entries)
I170517 07:32:21.350903 30728 storage/replica_raftstorage.go:605  [s3,r1/3:/M{in-ax}] applied Raft snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:21.588638 30372 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:38397->127.0.0.1:41618: read: connection reset by peer
I170517 07:32:21.589057 30201 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:52527->127.0.0.1:53344: use of closed network connection
--- PASS: TestRefreshPendingCommands (1.72s)
    --- PASS: TestRefreshPendingCommands/#00 (0.79s)
    --- PASS: TestRefreshPendingCommands/#01 (0.92s)
=== RUN   TestStoreRangeUpReplicate
I170517 07:32:21.645820 30817 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:44472" > attrs:<> locality:<>
W170517 07:32:21.660801 30817 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:21.662207 30884 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:44472
I170517 07:32:21.671334 30817 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:56209" > attrs:<> locality:<>
W170517 07:32:21.688493 30817 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:21.689991 30975 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:44472
I170517 07:32:21.711657 30817 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:51254" > attrs:<> locality:<>
I170517 07:32:21.750505 30817 storage/client_test.go:406  gossip network initialized
I170517 07:32:21.751449 30817 storage/replica_raftstorage.go:416  [replicate,s1,r1/1:/M{in-ax}] generated preemptive snapshot 38b4c13e at index 15
I170517 07:32:21.763128 31180 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=38b4c13e, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:21.763299 30817 storage/store.go:3345  [replicate,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 8.0 MiB/sec, 8ms
I170517 07:32:21.764485 31180 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:21.767113 30817 storage/replica_command.go:3538  [replicate,s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:21.776239 31184 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:21.783638 30966 storage/replica_raftstorage.go:416  [replicate,s1,r1/1:/M{in-ax}] generated preemptive snapshot c6bd4ca1 at index 17
I170517 07:32:21.786397 30966 storage/store.go:3345  [replicate,s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 8.0 MiB/sec, 2ms
I170517 07:32:21.787500 31206 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=c6bd4ca1, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:32:21.789089 31206 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:21.792125 30966 storage/replica_command.go:3538  [replicate,s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:21.793015 30864 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/queue.go:523
E170517 07:32:21.798341 30966 storage/queue.go:634  [replicate,s1,r1/1:/M{in-ax}] change replicas of r1 failed: failed to send RPC: sending to all 2 replicas failed; last error: failed to send RPC: store is stopped
I170517 07:32:21.810626 30337 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:44472->127.0.0.1:35541: use of closed network connection
I170517 07:32:21.810839 30840 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:56209->127.0.0.1:60800: use of closed network connection
--- PASS: TestStoreRangeUpReplicate (0.29s)
=== RUN   TestStoreRangeCorruptionChangeReplicas
I170517 07:32:21.921085 31210 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:21.946162 31210 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:44156" > attrs:<> locality:<>
W170517 07:32:21.995793 31210 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:21.998270 31366 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:44156
I170517 07:32:22.012972 31210 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:53638" > attrs:<> locality:<>
W170517 07:32:22.026349 31210 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:22.027584 31353 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:44156
I170517 07:32:22.035682 31210 storage/store.go:1252  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I170517 07:32:22.035934 31210 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:55150" > attrs:<> locality:<>
W170517 07:32:22.086397 31210 gossip/gossip.go:1196  [n4] no incoming or outgoing connections
I170517 07:32:22.096430 31591 gossip/client.go:131  [n4] started gossip client to 127.0.0.1:44156
I170517 07:32:22.102021 31210 storage/store.go:1252  [n4,s4]: failed initial metrics computation: [n4,s4]: system config not yet available
I170517 07:32:22.118216 31210 gossip/gossip.go:297  [n4] NodeDescriptor set to node_id:4 address:<network_field:"tcp" address_field:"127.0.0.1:44269" > attrs:<> locality:<>
W170517 07:32:22.146355 31210 gossip/gossip.go:1196  [n5] no incoming or outgoing connections
I170517 07:32:22.162081 31384 gossip/client.go:131  [n5] started gossip client to 127.0.0.1:44156
I170517 07:32:22.172215 31210 gossip/gossip.go:297  [n5] NodeDescriptor set to node_id:5 address:<network_field:"tcp" address_field:"127.0.0.1:45665" > attrs:<> locality:<>
I170517 07:32:22.174296 31472 gossip/server.go:285  [n1] refusing gossip from node 5 (max 3 conns); forwarding to 3 ({tcp 127.0.0.1:55150})
I170517 07:32:22.174943 31384 gossip/client.go:136  [n5] closing client to node 1 (127.0.0.1:44156): received forward from node 1 to 3 (127.0.0.1:55150)
I170517 07:32:22.178091 31721 gossip/gossip.go:1210  [n5] node has connected to cluster via gossip
I170517 07:32:22.180486 30636 gossip/client.go:131  [n5] started gossip client to 127.0.0.1:55150
W170517 07:32:22.193126 31210 gossip/gossip.go:1196  [n6] no incoming or outgoing connections
I170517 07:32:22.195680 31823 gossip/client.go:131  [n6] started gossip client to 127.0.0.1:44156
I170517 07:32:22.196703 30640 gossip/server.go:285  [n1] refusing gossip from node 6 (max 3 conns); forwarding to 4 ({tcp 127.0.0.1:44269})
I170517 07:32:22.215838 31823 gossip/client.go:136  [n6] closing client to node 1 (127.0.0.1:44156): received forward from node 1 to 4 (127.0.0.1:44269)
I170517 07:32:22.217038 31869 gossip/gossip.go:1210  [n6] node has connected to cluster via gossip
I170517 07:32:22.219712 30641 gossip/client.go:131  [n6] started gossip client to 127.0.0.1:44269
I170517 07:32:22.232488 31210 gossip/gossip.go:297  [n6] NodeDescriptor set to node_id:6 address:<network_field:"tcp" address_field:"127.0.0.1:57755" > attrs:<> locality:<>
I170517 07:32:22.293431 31210 storage/client_test.go:406  gossip network initialized
I170517 07:32:22.294384 31210 storage/replica_raftstorage.go:416  [replicate,s1,r1/1:/M{in-ax}] generated preemptive snapshot 19deab0f at index 18
I170517 07:32:22.296718 31210 storage/store.go:3345  [replicate,s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 8, rate-limit: 8.0 MiB/sec, 2ms
I170517 07:32:22.297755 32040 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 18 (id=19deab0f, encoded size=7035, 1 rocksdb batches, 8 log entries)
I170517 07:32:22.299191 32040 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:22.303382 31210 storage/replica_command.go:3538  [replicate,s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:22.309493 32045 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):2: [(n1,s1):1 (n3,s3):2]
I170517 07:32:22.317042 31210 storage/replica_raftstorage.go:416  [replicate,s1,r1/1:/M{in-ax}] generated preemptive snapshot f6baa2e5 at index 20
I170517 07:32:22.319997 31210 storage/store.go:3345  [replicate,s1,r1/1:/M{in-ax}] streamed snapshot to (n6,s6):?: kv pairs: 39, log entries: 10, rate-limit: 8.0 MiB/sec, 2ms
I170517 07:32:22.320839 32085 storage/replica_raftstorage.go:597  [s6,r1/?:{-}] applying preemptive snapshot at index 20 (id=f6baa2e5, encoded size=8595, 1 rocksdb batches, 10 log entries)
I170517 07:32:22.323678 32085 storage/replica_raftstorage.go:605  [s6,r1/?:/M{in-ax}] applied preemptive snapshot in 3ms [clear=0ms batch=0ms entries=2ms commit=0ms]
I170517 07:32:22.334067 31210 storage/replica_command.go:3538  [replicate,s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n6,s6):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):2, next=3]
I170517 07:32:22.366514 32100 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:22.375363 32115 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n6,s6):3: [(n1,s1):1 (n3,s3):2 (n6,s6):3]
E170517 07:32:22.387117 31502 storage/replica.go:4559  [s3,r1/2:/M{in-ax}] stalling replica due to: boom
I170517 07:32:22.398136 31210 storage/replica_command.go:3538  [replicate,s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):2, (n6,s6):3, next=4]
W170517 07:32:22.405401 32099 storage/store.go:3160  [s1] got error from r1, replica (n3,s3):2: replica corruption (processed=true): boom
W170517 07:32:22.408149 32099 storage/store.go:3160  [s1] got error from r1, replica (n3,s3):2: replica corruption (processed=true): boom
W170517 07:32:22.409427 32099 storage/store.go:3160  [s1] got error from r1, replica (n3,s3):2: replica corruption (processed=true): boom
I170517 07:32:22.410258 32109 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):2: [(n1,s1):1 (n6,s6):3]
W170517 07:32:22.417275 32099 storage/store.go:3160  [s1] got error from r1, replica (n3,s3):2: replica corruption (processed=true): boom
W170517 07:32:22.419707 32099 storage/store.go:3160  [s1] got error from r1, replica (n3,s3):2: replica corruption (processed=true): boom
I170517 07:32:22.423988 31210 storage/replica_raftstorage.go:416  [replicate,s1,r1/1:/M{in-ax}] generated preemptive snapshot aeff02f2 at index 25
I170517 07:32:22.426581 31210 storage/store.go:3345  [replicate,s1,r1/1:/M{in-ax}] streamed snapshot to (n5,s5):?: kv pairs: 46, log entries: 15, rate-limit: 8.0 MiB/sec, 2ms
I170517 07:32:22.434604 32124 storage/replica_raftstorage.go:597  [s5,r1/?:{-}] applying preemptive snapshot at index 25 (id=aeff02f2, encoded size=12273, 1 rocksdb batches, 15 log entries)
I170517 07:32:22.436912 32124 storage/replica_raftstorage.go:605  [s5,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:22.440476 31210 storage/replica_command.go:3538  [replicate,s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n5,s5):4): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n6,s6):3, next=4]
I170517 07:32:22.454113 32090 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n5,s5):4: [(n1,s1):1 (n6,s6):3 (n5,s5):4]
E170517 07:32:22.487098 31884 storage/replica.go:4559  [s6,r1/3:/M{in-ax}] stalling replica due to: boom
W170517 07:32:22.488549 32120 storage/store.go:3160  [s1] got error from r1, replica (n6,s6):3: replica corruption (processed=true): boom
W170517 07:32:22.493588 32120 storage/store.go:3160  [s1] got error from r1, replica (n6,s6):3: replica corruption (processed=true): boom
I170517 07:32:22.501788 31210 storage/replica_command.go:3538  [replicate,s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n6,s6):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n6,s6):3, (n5,s5):4, next=5]
W170517 07:32:22.502916 32120 storage/store.go:3160  [s1] got error from r1, replica (n6,s6):3: replica corruption (processed=true): boom
W170517 07:32:22.505158 32120 storage/store.go:3160  [s1] got error from r1, replica (n6,s6):3: replica corruption (processed=true): boom
W170517 07:32:22.506766 32120 storage/store.go:3160  [s1] got error from r1, replica (n6,s6):3: replica corruption (processed=true): boom
I170517 07:32:22.510785 32192 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n6,s6):3: [(n1,s1):1 (n5,s5):4]
W170517 07:32:22.519172 32120 storage/store.go:3160  [s1] got error from r1, replica (n6,s6):3: replica corruption (processed=true): boom
W170517 07:32:22.526919 32120 storage/store.go:3160  [s1] got error from r1, replica (n6,s6):3: replica corruption (processed=true): boom
I170517 07:32:22.530207 31210 storage/replica_raftstorage.go:416  [replicate,s1,r1/1:/M{in-ax}] generated preemptive snapshot c0ae6495 at index 30
I170517 07:32:22.532842 31210 storage/store.go:3345  [replicate,s1,r1/1:/M{in-ax}] streamed snapshot to (n4,s4):?: kv pairs: 53, log entries: 20, rate-limit: 8.0 MiB/sec, 2ms
I170517 07:32:22.533993 32212 storage/replica_raftstorage.go:597  [s4,r1/?:{-}] applying preemptive snapshot at index 30 (id=c0ae6495, encoded size=15951, 1 rocksdb batches, 20 log entries)
I170517 07:32:22.536204 32212 storage/replica_raftstorage.go:605  [s4,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:22.540145 31210 storage/replica_command.go:3538  [replicate,s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n4,s4):5): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n5,s5):4, next=5]
I170517 07:32:22.552015 32216 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n4,s4):5: [(n1,s1):1 (n5,s5):4 (n4,s4):5]
E170517 07:32:22.697332 31631 storage/replica.go:4559  [s4,r1/?:/M{in-ax}] stalling replica due to: boom
E170517 07:32:22.697947 31631 storage/replica.go:4559  [s4,r1/?:/M{in-ax}] stalling replica due to: boom
E170517 07:32:22.699374 31631 storage/replica.go:4559  [s4,r1/5:/M{in-ax}] stalling replica due to: boom
W170517 07:32:22.700224 32247 storage/store.go:3160  [s1] got error from r1, replica (n4,s4):5: replica corruption (processed=true): boom
W170517 07:32:22.700651 32247 storage/store.go:3160  [s1] got error from r1, replica (n4,s4):5: replica corruption (processed=true): boom
W170517 07:32:22.757874 32247 storage/store.go:3160  [s1] got error from r1, replica (n4,s4):5: replica corruption (processed=true): boom
W170517 07:32:22.760512 32247 storage/store.go:3160  [s1] got error from r1, replica (n4,s4):5: replica corruption (processed=true): boom
W170517 07:32:22.780453 32247 storage/store.go:3160  [s1] got error from r1, replica (n4,s4):5: replica corruption (processed=true): boom
W170517 07:32:22.783214 32247 storage/store.go:3160  [s1] got error from r1, replica (n4,s4):5: replica corruption (processed=true): boom
W170517 07:32:22.800629 32247 storage/store.go:3160  [s1] got error from r1, replica (n4,s4):5: replica corruption (processed=true): boom
W170517 07:32:22.834597 32247 storage/store.go:3160  [s1] got error from r1, replica (n4,s4):5: replica corruption (processed=true): boom
W170517 07:32:22.837552 32247 storage/store.go:3160  [s1] got error from r1, replica (n4,s4):5: replica corruption (processed=true): boom
I170517 07:32:22.842226 31210 storage/replica_command.go:3538  [replicate,s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n4,s4):5): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n5,s5):4, (n4,s4):5, next=6]
W170517 07:32:22.846555 32247 storage/store.go:3160  [s1] got error from r1, replica (n4,s4):5: replica corruption (processed=true): boom
W170517 07:32:22.851350 32247 storage/store.go:3160  [s1] got error from r1, replica (n4,s4):5: replica corruption (processed=true): boom
I170517 07:32:22.852816 32169 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n4,s4):5: [(n1,s1):1 (n5,s5):4]
W170517 07:32:22.860147 32247 storage/store.go:3160  [s1] got error from r1, replica (n4,s4):5: replica corruption (processed=true): boom
W170517 07:32:22.863623 32247 storage/store.go:3160  [s1] got error from r1, replica (n4,s4):5: replica corruption (processed=true): boom
I170517 07:32:22.869972 31210 storage/replica_raftstorage.go:416  [replicate,s1,r1/1:/M{in-ax}] generated preemptive snapshot 7a08e7a2 at index 38
I170517 07:32:22.872710 31210 storage/store.go:3345  [replicate,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 63, log entries: 28, rate-limit: 8.0 MiB/sec, 2ms
I170517 07:32:22.879317 32171 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 38 (id=7a08e7a2, encoded size=21093, 1 rocksdb batches, 28 log entries)
I170517 07:32:22.883262 32171 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 4ms [clear=0ms batch=0ms entries=3ms commit=0ms]
I170517 07:32:22.891944 31210 storage/replica_command.go:3538  [replicate,s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):6): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n5,s5):4, next=6]
I170517 07:32:22.899741 32324 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):6: [(n1,s1):1 (n5,s5):4 (n2,s2):6]
W170517 07:32:22.901499 32247 storage/store.go:3160  [s1] got error from r1, replica (n4,s4):5: replica corruption (processed=true): boom
I170517 07:32:22.915014 32264 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/client_test.go:510
1      kv/txn_coord_sender.go:981
I170517 07:32:22.915516 32264 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/client_test.go:510
W170517 07:32:22.919048 32103 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):6: no handler registered for (n1,s1):1
W170517 07:32:22.919644 32102 storage/store.go:3156  [s2] raft error: node 1 claims to not contain store 1 for replica (n1,s1):1: store 1 was not found
W170517 07:32:22.919843 32100 storage/raft_transport.go:442  raft transport stream to node 1 failed: store 1 was not found
I170517 07:32:22.955273 31821 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:57755->127.0.0.1:54862: use of closed network connection
I170517 07:32:22.955723 31862 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:22.956086 31423 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:55150->127.0.0.1:40217: use of closed network connection
W170517 07:32:22.956230 32245 storage/raft_transport.go:442  raft transport stream to node 4 failed: EOF
I170517 07:32:22.956439 31605 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:44269->127.0.0.1:32882: use of closed network connection
I170517 07:32:22.956571 31364 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:53638->127.0.0.1:42943: use of closed network connection
W170517 07:32:22.956797 32338 storage/raft_transport.go:442  raft transport stream to node 2 failed: EOF
I170517 07:32:22.956864 31607 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:22.957202 31253 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
W170517 07:32:22.958947 32065 storage/raft_transport.go:442  raft transport stream to node 3 failed: EOF
I170517 07:32:22.959136 31865 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:57755: getsockopt: connection refused"; Reconnecting to {127.0.0.1:57755 <nil>}
I170517 07:32:22.959283 31865 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
I170517 07:32:22.959752 31716 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:45665->127.0.0.1:49483: use of closed network connection
I170517 07:32:22.960609 31610 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:44269: getsockopt: connection refused"; Reconnecting to {127.0.0.1:44269 <nil>}
I170517 07:32:22.960713 31610 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestStoreRangeCorruptionChangeReplicas (1.09s)
=== RUN   TestUnreplicateFirstRange
I170517 07:32:23.019796 32311 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:54766" > attrs:<> locality:<>
W170517 07:32:23.034510 32311 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:23.044405 32311 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:32:23.044584 32396 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:54766
I170517 07:32:23.044669 32311 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:58241" > attrs:<> locality:<>
W170517 07:32:23.113657 32311 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:23.116779 32333 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:54766
I170517 07:32:23.134253 32311 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:52200" > attrs:<> locality:<>
I170517 07:32:23.147763 32699 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot bacdee14 at index 15
I170517 07:32:23.152323 32699 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 4ms
I170517 07:32:23.159953 32724 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=bacdee14, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:23.161565 32724 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:23.165068 32699 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:23.171285 32740 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:23.244829 32760 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:23.341892 32787 storage/replica_command.go:3538  [s2,r1/2:/M{in-ax}] change replicas (REMOVE_REPLICA (n1,s1):1): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:23.348044 32511 storage/replica.go:2590  [s2,r1/2:/M{in-ax}] proposing REMOVE_REPLICA (n1,s1):1: [(n2,s2):2]
I170517 07:32:23.355031 32391 storage/store.go:2139  [replicaGC,s1,r1/1:/M{in-ax}] removing replica
I170517 07:32:23.355549 32391 storage/replica.go:684  [replicaGC,s1,r1/1:/M{in-ax}] removed 33 (25+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:23.356306 32709 storage/replica_raftstorage.go:416  [s2,r1/2:/M{in-ax}] generated preemptive snapshot e744e34e at index 22
I170517 07:32:23.359238 32709 storage/store.go:3345  [s2,r1/2:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 40, log entries: 12, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:23.364803 32602 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 22 (id=e744e34e, encoded size=9585, 1 rocksdb batches, 12 log entries)
I170517 07:32:23.366486 32602 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:23.369747 32709 storage/replica_command.go:3538  [s2,r1/2:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n2,s2):2, next=3]
I170517 07:32:23.374313 32605 storage/replica.go:2590  [s2,r1/2:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n2,s2):2 (n3,s3):3]
W170517 07:32:23.395260 32762 storage/store.go:3160  [s2] got error from r1, replica (n1,s1):1: raft group deleted
--- PASS: TestUnreplicateFirstRange (0.71s)
=== RUN   TestChangeReplicasDescriptorInvariant
I170517 07:32:23.711275 32821 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:23.711516 32821 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:40172" > attrs:<> locality:<>
W170517 07:32:23.728301 32821 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:23.731017 32717 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:40172
I170517 07:32:23.738913 32821 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:59205" > attrs:<> locality:<>
W170517 07:32:23.751579 32821 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:23.754303 32990 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:40172
I170517 07:32:23.760942 32821 storage/store.go:1252  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I170517 07:32:23.761196 32821 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:39760" > attrs:<> locality:<>
I170517 07:32:23.782714 32821 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 52e80edc at index 15
I170517 07:32:23.785019 32821 storage/store.go:3345  streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:23.785937 33082 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=52e80edc, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:23.787336 33082 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:23.789605 32821 storage/replica_command.go:3538  change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:23.794005 33086 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:23.801832 32821 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot a9288ace at index 17
I170517 07:32:23.803975 32821 storage/store.go:3345  streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:23.804814 33176 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=a9288ace, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:32:23.806348 33176 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:23.808540 32821 storage/replica_command.go:3538  change replicas (ADD_REPLICA (n3,s3):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:23.811797 32821 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot de9addca at index 17
I170517 07:32:23.815637 32821 storage/store.go:3345  streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:23.819164 32821 storage/replica_command.go:3538  change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:23.822456 33194 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:23.830375 33220 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:23.875727 33074 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:39760->127.0.0.1:43503: use of closed network connection
I170517 07:32:23.876124 32775 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:40172->127.0.0.1:33280: use of closed network connection
W170517 07:32:23.876523 33194 storage/raft_transport.go:442  raft transport stream to node 1 failed: EOF
I170517 07:32:23.881286 33049 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:23.881870 33052 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:39760: operation was canceled"; Reconnecting to {127.0.0.1:39760 <nil>}
I170517 07:32:23.883136 33052 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestChangeReplicasDescriptorInvariant (0.22s)
=== RUN   TestProgressWithDownNode
I170517 07:32:23.925676 32944 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:23.927689 32944 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:33304" > attrs:<> locality:<>
W170517 07:32:23.959971 32944 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:23.961545 33368 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:33304
I170517 07:32:23.970549 32944 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:47756" > attrs:<> locality:<>
W170517 07:32:23.998608 32944 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:24.020206 32944 storage/store.go:1252  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I170517 07:32:24.024263 32944 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:51532" > attrs:<> locality:<>
I170517 07:32:24.027236 33251 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:33304
I170517 07:32:24.071974 33602 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot dceed1a6 at index 15
I170517 07:32:24.075215 33602 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:24.076437 33338 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=dceed1a6, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:24.077893 33338 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:24.080629 33602 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:24.085892 33607 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:24.092705 33339 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot f3facd78 at index 17
I170517 07:32:24.119338 33339 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 26ms
I170517 07:32:24.120331 33342 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=f3facd78, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:32:24.125126 33342 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:24.128434 33339 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:24.136468 33618 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:24.161551 33354 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
W170517 07:32:24.354146 33352 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:24.354810 33617 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):2: store 2 was not found
W170517 07:32:24.355006 33615 storage/raft_transport.go:442  raft transport stream to node 2 failed: store 2 was not found
I170517 07:32:24.357308 33255 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:24.358173 33640 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:24.358796 33638 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):2: store 2 was not found
I170517 07:32:24.388440 33224 storage/raft_transport.go:436  raft transport stream to node 2 established
I170517 07:32:24.439278 33222 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:51532->127.0.0.1:45460: use of closed network connection
I170517 07:32:24.440974 33065 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:47756->127.0.0.1:39428: use of closed network connection
I170517 07:32:24.441341 33208 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:33304->127.0.0.1:35791: use of closed network connection
--- PASS: TestProgressWithDownNode (0.55s)
=== RUN   TestReplicateRestartAfterTruncationWithRemoveAndReAdd
I170517 07:32:24.487084 33600 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:24.487333 33600 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:41739" > attrs:<> locality:<>
W170517 07:32:24.506938 33600 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:24.510277 33744 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:41739
I170517 07:32:24.519410 33600 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:38295" > attrs:<> locality:<>
W170517 07:32:24.534741 33600 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:24.536064 33988 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:41739
I170517 07:32:24.548634 33600 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:34437" > attrs:<> locality:<>
I170517 07:32:24.567076 33934 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 387483e0 at index 15
I170517 07:32:24.570584 33934 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:24.572808 33937 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=387483e0, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:24.574031 33937 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:24.576269 33934 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:24.585074 34117 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:24.591312 34101 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 59affc10 at index 17
I170517 07:32:24.593670 34101 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:24.594736 34104 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=59affc10, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:32:24.596129 34104 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:24.598040 34101 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:24.603192 34108 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:24.614823 34128 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:24.772136 33233 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:32:24.780951 34112 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):2: [(n1,s1):1 (n3,s3):3]
W170517 07:32:24.794172 33229 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:24.794558 33229 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:24.794743 34022 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):2: store 2 was not found
W170517 07:32:24.794852 33229 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:24.794941 34107 storage/raft_transport.go:442  raft transport stream to node 2 failed: store 2 was not found
W170517 07:32:24.795101 33229 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
I170517 07:32:24.849141 33600 storage/store.go:2139  [replicaGC,s2,r1/2:/M{in-ax}] removing replica
I170517 07:32:24.849750 33600 storage/replica.go:684  [replicaGC,s2,r1/2:/M{in-ax}] removed 34 (26+8) keys in 0ms [clear=0ms commit=0ms]
W170517 07:32:24.852779 33916 storage/replica.go:4351  [s2,r1/2:/M{in-ax}] could not acquire lease for range gossip: r1 was not found
W170517 07:32:24.852915 33916 storage/store.go:1339  [s2,r1/2:/M{in-ax}] could not gossip system config: r1 was not found
W170517 07:32:24.853093 33917 storage/replica.go:4351  [s2,r1/2:/M{in-ax}] could not acquire lease for range gossip: r1 was not found
W170517 07:32:24.853218 33917 storage/store.go:1339  [s2,r1/2:/M{in-ax}] could not gossip node liveness: r1 was not found
W170517 07:32:24.854658 33915 storage/replica.go:4351  [s2,r1/2:/M{in-ax}] could not acquire lease for range gossip: r1 was not found
W170517 07:32:24.854775 33915 storage/store.go:1339  [s2,r1/2:/M{in-ax}] could not gossip first range descriptor: r1 was not found
I170517 07:32:24.866616 34245 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 8563e7e1 at index 26
I170517 07:32:24.869396 34245 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 45, log entries: 4, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:24.874221 34166 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 26 (id=8563e7e1, encoded size=6415, 1 rocksdb batches, 4 log entries)
I170517 07:32:24.875438 34166 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:24.879977 34245 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):4): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=4]
I170517 07:32:24.888054 34094 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):4: [(n1,s1):1 (n3,s3):3 (n2,s2):4]
W170517 07:32:25.180322 34281 storage/raft_transport.go:442  raft transport stream to node 2 failed: EOF
I170517 07:32:25.183451 34019 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:34437->127.0.0.1:48528: read: connection reset by peer
I170517 07:32:25.183807 33721 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:41739->127.0.0.1:51804: read: connection reset by peer
W170517 07:32:25.186400 34134 storage/raft_transport.go:442  raft transport stream to node 3 failed: rpc error: code = Internal desc = transport is closing
--- PASS: TestReplicateRestartAfterTruncationWithRemoveAndReAdd (0.74s)
=== RUN   TestReplicateRestartAfterTruncation
I170517 07:32:25.218840 34246 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:25.219113 34246 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:40568" > attrs:<> locality:<>
W170517 07:32:25.241587 34246 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:25.251127 34246 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:32:25.251443 34246 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:52030" > attrs:<> locality:<>
I170517 07:32:25.257450 34426 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:40568
W170517 07:32:25.281064 34246 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:25.282407 34156 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:40568
I170517 07:32:25.311177 34246 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:34152" > attrs:<> locality:<>
I170517 07:32:25.321330 34137 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 6dd51a9f at index 15
I170517 07:32:25.323383 34137 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 1ms
I170517 07:32:25.324486 34433 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=6dd51a9f, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:25.326135 34433 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:25.328123 34137 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:25.335165 34142 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:25.344332 34161 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 8eec099d at index 17
I170517 07:32:25.349287 34161 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 4ms
I170517 07:32:25.351881 34525 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=8eec099d, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:32:25.353566 34525 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:25.355719 34161 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:25.360192 34666 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:25.370452 34670 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
W170517 07:32:25.549951 34665 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:25.550567 34663 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):2: store 2 was not found
W170517 07:32:25.550775 34661 storage/raft_transport.go:442  raft transport stream to node 2 failed: store 2 was not found
I170517 07:32:25.551784 34638 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:25.553576 34674 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:25.554180 34397 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):2: store 2 was not found
I170517 07:32:25.558978 34639 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:25.561548 34527 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:25.562098 34641 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):2: store 2 was not found
I170517 07:32:25.575094 34770 storage/raft_transport.go:436  raft transport stream to node 2 established
I170517 07:32:25.620322 34676 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/client_test.go:510
W170517 07:32:25.623244 34645 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n1,s1):1
W170517 07:32:25.623681 34645 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n1,s1):?
W170517 07:32:25.623973 34668 storage/store.go:3156  [s2] raft error: node 1 claims to not contain store 1 for replica (n1,s1):1: store 1 was not found
W170517 07:32:25.624189 34666 storage/raft_transport.go:442  raft transport stream to node 1 failed: store 1 was not found
I170517 07:32:25.634734 34154 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:34152->127.0.0.1:39531: use of closed network connection
I170517 07:32:25.634848 34536 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:25.635239 34408 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
W170517 07:32:25.635391 34651 storage/raft_transport.go:442  raft transport stream to node 3 failed: EOF
I170517 07:32:25.635613 34393 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:52030->127.0.0.1:46376: use of closed network connection
I170517 07:32:25.635868 34306 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:40568->127.0.0.1:43931: use of closed network connection
I170517 07:32:25.636711 34539 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:34152: operation was canceled"; Reconnecting to {127.0.0.1:34152 <nil>}
I170517 07:32:25.636911 34539 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestReplicateRestartAfterTruncation (0.45s)
=== RUN   TestReplicateAddAndRemove
I170517 07:32:25.693650 34654 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:25.693968 34654 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:45010" > attrs:<> locality:<>
W170517 07:32:25.712390 34654 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:25.713949 34907 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:45010
I170517 07:32:25.725521 34654 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:45600" > attrs:<> locality:<>
W170517 07:32:25.739166 34654 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:25.740840 35060 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:45010
I170517 07:32:25.757735 34654 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:52922" > attrs:<> locality:<>
W170517 07:32:25.781454 34654 gossip/gossip.go:1196  [n4] no incoming or outgoing connections
I170517 07:32:25.783025 34806 gossip/client.go:131  [n4] started gossip client to 127.0.0.1:45010
I170517 07:32:25.808972 34654 gossip/gossip.go:297  [n4] NodeDescriptor set to node_id:4 address:<network_field:"tcp" address_field:"127.0.0.1:43878" > attrs:<> locality:<>
I170517 07:32:25.825743 35188 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot a31a275d at index 16
I170517 07:32:25.828081 35188 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n4,s4):?: kv pairs: 34, log entries: 6, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:25.830050 35285 storage/replica_raftstorage.go:597  [s4,r1/?:{-}] applying preemptive snapshot at index 16 (id=a31a275d, encoded size=6059, 1 rocksdb batches, 6 log entries)
I170517 07:32:25.831337 35285 storage/replica_raftstorage.go:605  [s4,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:25.835169 35188 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n4,s4):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:25.839658 34814 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n4,s4):2: [(n1,s1):1 (n4,s4):2]
I170517 07:32:25.848508 35190 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 291df129 at index 18
I170517 07:32:25.853091 35190 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 37, log entries: 8, rate-limit: 2.0 MiB/sec, 4ms
I170517 07:32:25.854029 35304 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 18 (id=291df129, encoded size=7619, 1 rocksdb batches, 8 log entries)
I170517 07:32:25.855663 35304 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:25.859309 35190 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n4,s4):2, next=3]
I170517 07:32:25.863897 35314 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:25.870161 35195 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):3: [(n1,s1):1 (n4,s4):2 (n2,s2):3]
I170517 07:32:25.960356 35334 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot def8208e at index 21
I170517 07:32:25.963094 35334 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 41, log entries: 11, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:25.971435 35164 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 21 (id=def8208e, encoded size=9655, 1 rocksdb batches, 11 log entries)
I170517 07:32:25.976242 35164 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 5ms [clear=0ms batch=0ms entries=4ms commit=0ms]
I170517 07:32:25.978594 35334 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):4): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n4,s4):2, (n2,s2):3, next=4]
W170517 07:32:25.981109 34943 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):3
W170517 07:32:25.981684 35348 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):3: store 2 was not found
W170517 07:32:25.982980 35311 storage/raft_transport.go:442  raft transport stream to node 2 failed: store 2 was not found
I170517 07:32:25.986604 35167 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):4: [(n1,s1):1 (n4,s4):2 (n2,s2):3 (n3,s3):4]
I170517 07:32:25.988027 35168 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:25.996475 35318 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):3
W170517 07:32:25.996966 35318 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):?: no handler registered for (n2,s2):?
W170517 07:32:25.997677 35362 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):3: store 2 was not found
I170517 07:32:26.000836 35363 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:26.002171 35339 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):3
W170517 07:32:26.002815 35337 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):3: store 2 was not found
I170517 07:32:26.144324 35369 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:26.145064 35322 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):?: no handler registered for (n2,s2):?
W170517 07:32:26.145682 35372 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):?: store 2 was not found
I170517 07:32:26.194300 34944 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:26.195238 35377 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):?: no handler registered for (n2,s2):?
W170517 07:32:26.196478 35379 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):?: store 2 was not found
I170517 07:32:26.280145 35382 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n4,s4):2, (n2,s2):3, (n3,s3):4, next=5]
I170517 07:32:26.282756 35385 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:26.284564 35395 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):3
W170517 07:32:26.285163 35387 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):3: store 2 was not found
I170517 07:32:26.286644 35396 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:26.287515 35343 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):3
I170517 07:32:26.289235 35411 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):3: [(n1,s1):1 (n4,s4):2 (n3,s3):4]
W170517 07:32:26.293479 35343 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):3
W170517 07:32:26.293729 35343 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):3
W170517 07:32:26.294118 35398 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):3: store 2 was not found
I170517 07:32:26.341224 34654 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
I170517 07:32:26.355896 34654 storage/store.go:2139  [replicaGC,s2,r1/3:/M{in-ax}] removing replica
I170517 07:32:26.356436 34654 storage/replica.go:684  [replicaGC,s2,r1/3:/M{in-ax}] removed 35 (27+8) keys in 0ms [clear=0ms commit=0ms]
W170517 07:32:26.359233 35193 storage/raft_transport.go:257  unable to accept Raft message from (n4,s4):2: no handler registered for (n1,s1):1
W170517 07:32:26.359922 35316 storage/store.go:3156  [s4] raft error: node 1 claims to not contain store 1 for replica (n1,s1):1: store 1 was not found
W170517 07:32:26.361246 35314 storage/raft_transport.go:442  raft transport stream to node 1 failed: store 1 was not found
W170517 07:32:26.361795 35193 storage/raft_transport.go:257  unable to accept Raft message from (n4,s4):2: no handler registered for (n1,s1):1
I170517 07:32:26.363411 35415 storage/raft_transport.go:436  raft transport stream to node 1 established
W170517 07:32:26.364283 35527 storage/raft_transport.go:257  unable to accept Raft message from (n3,s3):4: no handler registered for (n1,s1):1
W170517 07:32:26.364610 35527 storage/raft_transport.go:257  unable to accept Raft message from (n3,s3):4: no handler registered for (n1,s1):1
W170517 07:32:26.365105 35390 storage/store.go:3156  [s3] raft error: node 1 claims to not contain store 1 for replica (n1,s1):1: store 1 was not found
W170517 07:32:26.365784 35527 storage/raft_transport.go:257  unable to accept Raft message from (n3,s3):4: no handler registered for (n1,s1):1
I170517 07:32:26.382461 35161 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:43878->127.0.0.1:59952: use of closed network connection
I170517 07:32:26.383528 35073 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
W170517 07:32:26.383675 35370 storage/raft_transport.go:442  raft transport stream to node 3 failed: EOF
I170517 07:32:26.383990 34970 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:52922->127.0.0.1:50728: use of closed network connection
I170517 07:32:26.384686 35172 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:43878: operation was canceled"; Reconnecting to {127.0.0.1:43878 <nil>}
I170517 07:32:26.384809 35172 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestReplicateAddAndRemove (0.83s)
=== RUN   TestReplicateRemoveAndAdd
I170517 07:32:26.494887 35391 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:26.495165 35391 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:44019" > attrs:<> locality:<>
W170517 07:32:26.519520 35391 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:26.532239 35391 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:32:26.532478 35391 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:46678" > attrs:<> locality:<>
I170517 07:32:26.536693 35666 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:44019
W170517 07:32:26.559310 35391 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:26.563899 35632 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:44019
I170517 07:32:26.587792 35391 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:53317" > attrs:<> locality:<>
W170517 07:32:26.625070 35391 gossip/gossip.go:1196  [n4] no incoming or outgoing connections
I170517 07:32:26.631109 35537 gossip/client.go:131  [n4] started gossip client to 127.0.0.1:44019
I170517 07:32:26.642878 35391 gossip/gossip.go:297  [n4] NodeDescriptor set to node_id:4 address:<network_field:"tcp" address_field:"127.0.0.1:57859" > attrs:<> locality:<>
I170517 07:32:26.666758 35505 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot cca95e63 at index 16
I170517 07:32:26.669218 35505 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n4,s4):?: kv pairs: 34, log entries: 6, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:26.670397 36022 storage/replica_raftstorage.go:597  [s4,r1/?:{-}] applying preemptive snapshot at index 16 (id=cca95e63, encoded size=6059, 1 rocksdb batches, 6 log entries)
I170517 07:32:26.672967 36022 storage/replica_raftstorage.go:605  [s4,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=2ms commit=0ms]
I170517 07:32:26.675090 35505 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n4,s4):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:26.679792 35976 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n4,s4):2: [(n1,s1):1 (n4,s4):2]
I170517 07:32:26.692354 36039 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 1c1adfd9 at index 18
I170517 07:32:26.696253 36042 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 18 (id=1c1adfd9, encoded size=7619, 1 rocksdb batches, 8 log entries)
I170517 07:32:26.697675 36042 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:26.698700 36039 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 37, log entries: 8, rate-limit: 2.0 MiB/sec, 6ms
I170517 07:32:26.700086 36039 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n4,s4):2, next=3]
I170517 07:32:26.705036 36050 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:26.713992 36024 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):3: [(n1,s1):1 (n4,s4):2 (n2,s2):3]
I170517 07:32:27.006974 35980 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n4,s4):2, (n2,s2):3, next=4]
W170517 07:32:27.010944 35758 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):3
W170517 07:32:27.011524 35756 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):3: store 2 was not found
W170517 07:32:27.012081 35754 storage/raft_transport.go:442  raft transport stream to node 2 failed: store 2 was not found
I170517 07:32:27.015528 36010 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:27.017898 35760 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):3
W170517 07:32:27.018436 36012 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):3: store 2 was not found
I170517 07:32:27.020220 36014 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):3: [(n1,s1):1 (n4,s4):2]
I170517 07:32:27.029220 36081 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot b0610227 at index 23
I170517 07:32:27.032698 36015 storage/raft_transport.go:436  raft transport stream to node 2 established
W170517 07:32:27.040395 36075 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):3
W170517 07:32:27.040709 36075 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):3
W170517 07:32:27.041282 36028 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):3: store 2 was not found
I170517 07:32:27.042594 36081 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 44, log entries: 13, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:27.044087 36100 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 23 (id=b0610227, encoded size=11219, 1 rocksdb batches, 13 log entries)
I170517 07:32:27.045507 36101 storage/raft_transport.go:436  raft transport stream to node 2 established
I170517 07:32:27.046002 36100 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
W170517 07:32:27.046467 36114 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):?: no handler registered for (n2,s2):?
W170517 07:32:27.047010 36103 storage/store.go:3156  [s1] raft error: node 2 claims to not contain store 2 for replica (n2,s2):?: store 2 was not found
I170517 07:32:27.049580 36081 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):4): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n4,s4):2, next=4]
I170517 07:32:27.060672 36130 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):4: [(n1,s1):1 (n4,s4):2 (n3,s3):4]
I170517 07:32:27.221788 35391 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
I170517 07:32:27.232685 35391 storage/store.go:2139  [replicaGC,s2,r1/3:/M{in-ax}] removing replica
I170517 07:32:27.236702 35391 storage/replica.go:684  [replicaGC,s2,r1/3:/M{in-ax}] removed 35 (27+8) keys in 1ms [clear=0ms commit=0ms]
W170517 07:32:27.246318 36067 storage/raft_transport.go:257  unable to accept Raft message from (n3,s3):4: no handler registered for (n1,s1):1
W170517 07:32:27.246638 36067 storage/raft_transport.go:257  unable to accept Raft message from (n3,s3):4: no handler registered for (n1,s1):1
W170517 07:32:27.247094 36052 storage/store.go:3156  [s3] raft error: node 1 claims to not contain store 1 for replica (n1,s1):1: store 1 was not found
W170517 07:32:27.247275 36050 storage/raft_transport.go:442  raft transport stream to node 1 failed: store 1 was not found
I170517 07:32:27.330569 35535 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:57859->127.0.0.1:59728: use of closed network connection
I170517 07:32:27.331416 35762 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:53317->127.0.0.1:34400: use of closed network connection
I170517 07:32:27.332687 35636 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:46678->127.0.0.1:48277: use of closed network connection
I170517 07:32:27.333002 35638 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:27.333744 35528 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:44019->127.0.0.1:40634: use of closed network connection
--- PASS: TestReplicateRemoveAndAdd (0.88s)
=== RUN   TestRaftHeartbeats
I170517 07:32:27.379957 36058 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:27.380237 36058 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:50681" > attrs:<> locality:<>
W170517 07:32:27.409008 36058 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:27.411535 36370 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:50681
I170517 07:32:27.428033 36058 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:48490" > attrs:<> locality:<>
W170517 07:32:27.457099 36058 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:27.459418 36138 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:50681
I170517 07:32:27.472407 36058 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:54576" > attrs:<> locality:<>
I170517 07:32:27.496145 36463 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 58c53712 at index 15
I170517 07:32:27.498241 36463 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:27.499040 36111 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=58c53712, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:27.500303 36111 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:27.502233 36463 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:27.507144 36627 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:27.513586 36601 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 5301bf61 at index 17
I170517 07:32:27.515673 36601 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:27.516542 36343 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=5301bf61, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:32:27.519961 36343 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 3ms [clear=0ms batch=0ms entries=2ms commit=0ms]
I170517 07:32:27.522189 36601 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:27.526702 36605 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:27.533527 36639 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:28.190576 36480 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:54576->127.0.0.1:53858: use of closed network connection
I170517 07:32:28.191019 36233 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:50681->127.0.0.1:43498: use of closed network connection
I170517 07:32:28.191849 36032 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:48490->127.0.0.1:38852: read: connection reset by peer
--- PASS: TestRaftHeartbeats (0.85s)
=== RUN   TestReportUnreachableHeartbeats
I170517 07:32:28.262195 36645 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:28.262605 36645 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:57669" > attrs:<> locality:<>
W170517 07:32:28.295577 36645 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:28.301536 36703 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:57669
I170517 07:32:28.308710 36645 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:49345" > attrs:<> locality:<>
W170517 07:32:28.323298 36645 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:28.331158 36858 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:57669
I170517 07:32:28.345208 36645 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:38558" > attrs:<> locality:<>
I170517 07:32:28.366053 36911 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 0c07b787 at index 15
I170517 07:32:28.369519 37010 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=0c07b787, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:28.372038 37010 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=2ms commit=0ms]
I170517 07:32:28.372846 36911 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 6ms
I170517 07:32:28.375854 36911 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:28.381986 37005 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:28.389208 37027 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 9c4943bc at index 17
I170517 07:32:28.391855 37027 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:28.392709 37008 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=9c4943bc, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:32:28.394117 37008 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:28.402906 37027 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:28.407699 36799 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:28.430725 37044 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:29.081550 37001 storage/node_liveness.go:352  [hb] heartbeat result is ambiguous (sending to all 3 replicas failed; last error: failed to send RPC: store is stopped, but RPC failure may have masked txn commit); retrying
W170517 07:32:29.082403 37001 storage/node_liveness.go:253  [hb] failed node liveness heartbeat: node unavailable; try another peer
I170517 07:32:29.100859 36779 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:49345->127.0.0.1:41968: use of closed network connection
I170517 07:32:29.101465 36780 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:29.103072 36464 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:57669->127.0.0.1:54785: use of closed network connection
I170517 07:32:29.104137 36686 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:38558->127.0.0.1:49875: use of closed network connection
I170517 07:32:29.104614 37041 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken read tcp 127.0.0.1:41978->127.0.0.1:49345: read: connection reset by peer.
I170517 07:32:29.105030 36767 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:29.108292 36783 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:49345: operation was canceled"; Reconnecting to {127.0.0.1:49345 <nil>}
I170517 07:32:29.108436 36783 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
I170517 07:32:29.122044 36914 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:38558: operation was canceled"; Reconnecting to {127.0.0.1:38558 <nil>}
I170517 07:32:29.122189 36914 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestReportUnreachableHeartbeats (0.92s)
=== RUN   TestReplicateAfterSplit
I170517 07:32:29.174170 36653 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:46445" > attrs:<> locality:<>
W170517 07:32:29.226076 36653 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:29.231481 37093 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:46445
I170517 07:32:29.253552 36653 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:37108" > attrs:<> locality:<>
I170517 07:32:29.297967 36653 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "m" [r2]
E170517 07:32:29.312698 37119 storage/queue.go:634  [replicate,s1,r1/1:{/Min-m}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:32:29.316954 37119 storage/queue.go:634  [replicate,s1,r2/1:{m-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:32:29.320554 37333 storage/replica_raftstorage.go:416  [s1,r2/1:{m-/Max}] generated preemptive snapshot 8cf689f6 at index 12
I170517 07:32:29.325728 37105 storage/replica_raftstorage.go:597  [s2,r2/?:{-}] applying preemptive snapshot at index 12 (id=8cf689f6, encoded size=3952, 1 rocksdb batches, 2 log entries)
I170517 07:32:29.326859 37105 storage/replica_raftstorage.go:605  [s2,r2/?:{m-/Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:29.345866 37333 storage/store.go:3345  [s1,r2/1:{m-/Max}] streamed snapshot to (n2,s2):?: kv pairs: 28, log entries: 2, rate-limit: 2.0 MiB/sec, 24ms
I170517 07:32:29.364211 37333 storage/replica_command.go:3538  [s1,r2/1:{m-/Max}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r2:{m-/Max} [(n1,s1):1, next=2]
W170517 07:32:29.373948 37349 storage/stores.go:218  range not contained in one range: [/Meta2/Max,"m\x00"), but have [/Min,"m")
I170517 07:32:29.396420 37339 storage/replica.go:2590  [s1,r2/1:{m-/Max}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:29.402533 37211 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:29.431799 37245 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
1      storage/client_test.go:510
W170517 07:32:29.437593 37210 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n2,s2):2
W170517 07:32:29.438825 37365 storage/raft_transport.go:476  no handler found for store 1 in response range_id:2 from_replica:<node_id:2 store_id:2 replica_id:2 > to_replica:<node_id:1 store_id:1 replica_id:1 > union:<error:<message:"store 2 was not found" transaction_restart:NONE origin_node:0 detail:<store_not_found:<store_id:2 > > now:<wall_time:0 logical:0 > > >
I170517 07:32:29.441058 36655 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:46445->127.0.0.1:59070: use of closed network connection
I170517 07:32:29.442316 37240 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:37108->127.0.0.1:52109: use of closed network connection
--- PASS: TestReplicateAfterSplit (0.33s)
=== RUN   TestReplicaRemovalCampaign
I170517 07:32:29.484535 37247 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:29.484805 37247 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:37240" > attrs:<> locality:<>
W170517 07:32:29.511549 37247 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:29.518981 37247 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:32:29.519184 37247 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:42717" > attrs:<> locality:<>
I170517 07:32:29.522524 37387 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:37240
I170517 07:32:29.541522 37389 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 99429904 at index 14
I170517 07:32:29.574385 37389 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 32, log entries: 4, rate-limit: 2.0 MiB/sec, 18ms
I170517 07:32:29.578888 37619 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 14 (id=99429904, encoded size=5089, 1 rocksdb batches, 4 log entries)
I170517 07:32:29.580064 37619 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:29.582095 37389 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:29.586794 37623 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:29.624281 37590 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:29.643671 37247 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "m" [r2]
E170517 07:32:29.708541 37376 storage/queue.go:634  [replicate,s1,r1/1:{/Min-m}] range requires a replication change, but lacks a quorum of live replicas (0/2)
E170517 07:32:29.709025 37376 storage/queue.go:634  [replicate,s1,r2/1:{m-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:32:29.733699 37247 storage/store.go:2139  removing replica
I170517 07:32:29.734210 37247 storage/replica.go:684  removed 28 (20+8) keys in 0ms [clear=0ms commit=0ms]
W170517 07:32:29.735089 37633 storage/store.go:3160  [s2] got error from r2, replica (n1,s1):1: raft group deleted
W170517 07:32:29.735936 37633 storage/store.go:3160  [s2] got error from r2, replica (n1,s1):1: raft group deleted
W170517 07:32:30.228456 37633 storage/store.go:3160  [s2] got error from r2, replica (n1,s1):1: raft group deleted
W170517 07:32:30.724207 37633 storage/store.go:3160  [s2] got error from r2, replica (n1,s1):1: raft group deleted
I170517 07:32:30.750380 37351 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:42717->127.0.0.1:40678: use of closed network connection
I170517 07:32:30.855311 37247 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:48844" > attrs:<> locality:<>
W170517 07:32:30.870384 37247 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:30.874371 37670 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:48844
I170517 07:32:30.897490 37247 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:54607" > attrs:<> locality:<>
I170517 07:32:30.907732 37784 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 849d04e9 at index 14
I170517 07:32:30.911392 37657 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 14 (id=849d04e9, encoded size=5089, 1 rocksdb batches, 4 log entries)
I170517 07:32:30.912486 37657 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:30.913165 37784 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 32, log entries: 4, rate-limit: 2.0 MiB/sec, 4ms
I170517 07:32:30.914548 37784 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:30.919018 37789 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:30.999035 37898 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:31.020861 37247 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "m" [r2]
E170517 07:32:31.040112 37646 storage/queue.go:634  [replicate,s1,r1/1:{/Min-m}] range requires a replication change, but lacks a quorum of live replicas (0/2)
E170517 07:32:31.056241 37646 storage/queue.go:634  [replicate,s1,r2/1:{m-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/2)
W170517 07:32:31.103380 37902 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n1,s1):?
W170517 07:32:31.103716 37902 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n1,s1):1
W170517 07:32:31.103904 37902 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n1,s1):1
W170517 07:32:31.104937 37900 storage/raft_transport.go:476  no handler found for store 2 in response range_id:0 from_replica:<node_id:1 store_id:1 replica_id:0 > to_replica:<node_id:2 store_id:2 replica_id:0 > union:<error:<message:"store 1 was not found" transaction_restart:NONE origin_node:0 detail:<store_not_found:<store_id:1 > > now:<wall_time:0 logical:0 > > >
W170517 07:32:31.105244 37900 storage/raft_transport.go:476  no handler found for store 2 in response range_id:2 from_replica:<node_id:1 store_id:1 replica_id:1 > to_replica:<node_id:2 store_id:2 replica_id:2 > union:<error:<message:"store 1 was not found" transaction_restart:NONE origin_node:0 detail:<store_not_found:<store_id:1 > > now:<wall_time:0 logical:0 > > >
W170517 07:32:31.105506 37900 storage/raft_transport.go:476  no handler found for store 2 in response range_id:2 from_replica:<node_id:1 store_id:1 replica_id:1 > to_replica:<node_id:2 store_id:2 replica_id:2 > union:<error:<message:"store 1 was not found" transaction_restart:NONE origin_node:0 detail:<store_not_found:<store_id:1 > > now:<wall_time:0 logical:0 > > >
I170517 07:32:31.128097 37600 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:54607->127.0.0.1:45051: use of closed network connection
I170517 07:32:31.128754 37794 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
--- PASS: TestReplicaRemovalCampaign (1.71s)
=== RUN   TestRaftAfterRemoveRange
I170517 07:32:31.221273 37681 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:31.221540 37681 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:43224" > attrs:<> locality:<>
W170517 07:32:31.308394 37681 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:31.313103 37503 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:43224
I170517 07:32:31.324176 37681 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:37843" > attrs:<> locality:<>
W170517 07:32:31.343277 37681 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:31.344990 37849 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:43224
I170517 07:32:31.380083 37681 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:60352" > attrs:<> locality:<>
I170517 07:32:31.410460 37681 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "b" [r2]
E170517 07:32:31.422759 38032 storage/queue.go:634  [replicate,s1,r1/1:{/Min-b}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:32:31.425042 38322 storage/replica_raftstorage.go:416  [s1,r2/1:{b-/Max}] generated preemptive snapshot 20bffd86 at index 11
E170517 07:32:31.428635 38032 storage/queue.go:634  [replicate,s1,r2/1:{b-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:32:31.429095 38322 storage/store.go:3345  [s1,r2/1:{b-/Max}] streamed snapshot to (n2,s2):?: kv pairs: 27, log entries: 1, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:31.430271 38117 storage/replica_raftstorage.go:597  [s2,r2/?:{-}] applying preemptive snapshot at index 11 (id=20bffd86, encoded size=3537, 1 rocksdb batches, 1 log entries)
I170517 07:32:31.431848 38117 storage/replica_raftstorage.go:605  [s2,r2/?:{b-/Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:31.441203 38322 storage/replica_command.go:3538  [s1,r2/1:{b-/Max}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r2:{b-/Max} [(n1,s1):1, next=2]
W170517 07:32:31.453763 38311 storage/stores.go:218  range not contained in one range: [/Meta2/Max,"b\x00"), but have [/Min,"b")
I170517 07:32:31.461981 38076 storage/replica.go:2590  [s1,r2/1:{b-/Max}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:31.467159 38078 storage/replica_raftstorage.go:416  [s1,r2/1:{b-/Max}] generated preemptive snapshot e7c2af1f at index 13
I170517 07:32:31.470958 38059 storage/replica_raftstorage.go:597  [s3,r2/?:{-}] applying preemptive snapshot at index 13 (id=e7c2af1f, encoded size=5052, 1 rocksdb batches, 3 log entries)
I170517 07:32:31.472144 38059 storage/replica_raftstorage.go:605  [s3,r2/?:{b-/Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:31.472930 38078 storage/store.go:3345  [s1,r2/1:{b-/Max}] streamed snapshot to (n3,s3):?: kv pairs: 29, log entries: 3, rate-limit: 2.0 MiB/sec, 5ms
I170517 07:32:31.480650 38122 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:31.482793 38078 storage/replica_command.go:3538  [s1,r2/1:{b-/Max}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r2:{b-/Max} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:31.506649 38340 storage/replica.go:2590  [s1,r2/1:{b-/Max}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:31.560111 38318 storage/replica_command.go:3538  [s1,r2/1:{b-/Max}] change replicas (REMOVE_REPLICA (n3,s3):3): read existing descriptor r2:{b-/Max} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:32:31.586176 38334 storage/replica.go:2590  [s1,r2/1:{b-/Max}] proposing REMOVE_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2]
I170517 07:32:31.596620 38245 storage/replica_command.go:3538  [s1,r2/1:{b-/Max}] change replicas (REMOVE_REPLICA (n2,s2):2): read existing descriptor r2:{b-/Max} [(n1,s1):1, (n2,s2):2, next=4]
I170517 07:32:31.602900 38187 storage/store.go:2139  [replicaGC,s3,r2/3:{b-/Max}] removing replica
I170517 07:32:31.603528 38187 storage/replica.go:684  [replicaGC,s3,r2/3:{b-/Max}] removed 28 (19+9) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:31.628011 37979 storage/replica.go:2590  [s1,r2/1:{b-/Max}] proposing REMOVE_REPLICA (n2,s2):2: [(n1,s1):1]
I170517 07:32:31.642755 38124 storage/store.go:3152  [s2,r2/2:{b-/Max}] added to replica GC queue (peer suggestion)
I170517 07:32:31.645856 38249 storage/replica_raftstorage.go:416  [s1,r1/1:{/Min-b}] generated preemptive snapshot f7b8bfe0 at index 26
I170517 07:32:31.651212 38249 storage/store.go:3345  [s1,r1/1:{/Min-b}] streamed snapshot to (n2,s2):?: kv pairs: 24, log entries: 16, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:31.683660 38386 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 26 (id=f7b8bfe0, encoded size=9087, 1 rocksdb batches, 16 log entries)
I170517 07:32:31.686187 38386 storage/replica_raftstorage.go:605  [s2,r1/?:{/Min-b}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:31.686660 38178 storage/store.go:2139  [replicaGC,s2,r2/2:{b-/Max}] removing replica
I170517 07:32:31.687246 38178 storage/replica.go:684  [replicaGC,s2,r2/2:{b-/Max}] removed 28 (19+9) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:31.689961 38249 storage/replica_command.go:3538  [s1,r1/1:{/Min-b}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:{/Min-b} [(n1,s1):1, next=2]
I170517 07:32:31.699137 38360 storage/replica.go:2590  [s1,r1/1:{/Min-b}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:31.781408 37681 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
--- PASS: TestRaftAfterRemoveRange (0.65s)
=== RUN   TestRaftRemoveRace
I170517 07:32:31.870006 37982 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:31.871783 37982 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:53032" > attrs:<> locality:<>
W170517 07:32:31.921954 37982 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:31.930501 37982 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:32:31.930755 37982 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:36501" > attrs:<> locality:<>
I170517 07:32:31.932606 37983 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:53032
W170517 07:32:31.959077 37982 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:31.961119 38255 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:53032
I170517 07:32:31.981992 37982 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:39790" > attrs:<> locality:<>
W170517 07:32:32.008437 37982 gossip/gossip.go:1196  [n4] no incoming or outgoing connections
I170517 07:32:32.012390 38770 gossip/client.go:131  [n4] started gossip client to 127.0.0.1:53032
I170517 07:32:32.039798 37982 gossip/gossip.go:297  [n4] NodeDescriptor set to node_id:4 address:<network_field:"tcp" address_field:"127.0.0.1:56411" > attrs:<> locality:<>
W170517 07:32:32.074965 37982 gossip/gossip.go:1196  [n5] no incoming or outgoing connections
I170517 07:32:32.077696 38766 gossip/server.go:285  [n1] refusing gossip from node 5 (max 3 conns); forwarding to 2 ({tcp 127.0.0.1:36501})
I170517 07:32:32.078625 38686 gossip/client.go:131  [n5] started gossip client to 127.0.0.1:53032
I170517 07:32:32.094966 38686 gossip/client.go:136  [n5] closing client to node 1 (127.0.0.1:53032): received forward from node 1 to 2 (127.0.0.1:36501)
I170517 07:32:32.095612 38781 gossip/gossip.go:1210  [n5] node has connected to cluster via gossip
I170517 07:32:32.096294 38414 gossip/client.go:131  [n5] started gossip client to 127.0.0.1:36501
I170517 07:32:32.116312 37982 gossip/gossip.go:297  [n5] NodeDescriptor set to node_id:5 address:<network_field:"tcp" address_field:"127.0.0.1:34960" > attrs:<> locality:<>
W170517 07:32:32.146267 37982 gossip/gossip.go:1196  [n6] no incoming or outgoing connections
I170517 07:32:32.148348 38965 gossip/client.go:131  [n6] started gossip client to 127.0.0.1:53032
I170517 07:32:32.149296 38892 gossip/server.go:285  [n1] refusing gossip from node 6 (max 3 conns); forwarding to 2 ({tcp 127.0.0.1:36501})
I170517 07:32:32.167324 37982 gossip/gossip.go:297  [n6] NodeDescriptor set to node_id:6 address:<network_field:"tcp" address_field:"127.0.0.1:42792" > attrs:<> locality:<>
I170517 07:32:32.168742 38965 gossip/client.go:136  [n6] closing client to node 1 (127.0.0.1:53032): received forward from node 1 to 2 (127.0.0.1:36501)
I170517 07:32:32.173054 39032 gossip/gossip.go:1210  [n6] node has connected to cluster via gossip
I170517 07:32:32.173602 39052 gossip/client.go:131  [n6] started gossip client to 127.0.0.1:36501
W170517 07:32:32.200333 37982 gossip/gossip.go:1196  [n7] no incoming or outgoing connections
I170517 07:32:32.208997 38971 gossip/client.go:131  [n7] started gossip client to 127.0.0.1:53032
I170517 07:32:32.210137 38910 gossip/server.go:285  [n1] refusing gossip from node 7 (max 3 conns); forwarding to 2 ({tcp 127.0.0.1:36501})
I170517 07:32:32.214210 37982 storage/store.go:1252  [n7,s7]: failed initial metrics computation: [n7,s7]: system config not yet available
I170517 07:32:32.214501 37982 gossip/gossip.go:297  [n7] NodeDescriptor set to node_id:7 address:<network_field:"tcp" address_field:"127.0.0.1:41324" > attrs:<> locality:<>
I170517 07:32:32.228197 38971 gossip/client.go:136  [n7] closing client to node 1 (127.0.0.1:53032): received forward from node 1 to 2 (127.0.0.1:36501)
I170517 07:32:32.238483 39179 gossip/gossip.go:1210  [n7] node has connected to cluster via gossip
I170517 07:32:32.241087 39160 gossip/client.go:131  [n7] started gossip client to 127.0.0.1:36501
W170517 07:32:32.262630 37982 gossip/gossip.go:1196  [n8] no incoming or outgoing connections
I170517 07:32:32.265567 39315 gossip/client.go:131  [n8] started gossip client to 127.0.0.1:53032
I170517 07:32:32.269327 39280 gossip/server.go:285  [n1] refusing gossip from node 8 (max 3 conns); forwarding to 4 ({tcp 127.0.0.1:56411})
I170517 07:32:32.280667 39315 gossip/client.go:136  [n8] closing client to node 1 (127.0.0.1:53032): received forward from node 1 to 4 (127.0.0.1:56411)
I170517 07:32:32.297140 39359 gossip/gossip.go:1210  [n8] node has connected to cluster via gossip
I170517 07:32:32.297870 39335 gossip/client.go:131  [n8] started gossip client to 127.0.0.1:56411
I170517 07:32:32.341842 37982 gossip/gossip.go:297  [n8] NodeDescriptor set to node_id:8 address:<network_field:"tcp" address_field:"127.0.0.1:35620" > attrs:<> locality:<>
W170517 07:32:32.374120 37982 gossip/gossip.go:1196  [n9] no incoming or outgoing connections
I170517 07:32:32.413454 39318 gossip/server.go:285  [n1] refusing gossip from node 9 (max 3 conns); forwarding to 2 ({tcp 127.0.0.1:36501})
I170517 07:32:32.426480 39305 gossip/client.go:131  [n9] started gossip client to 127.0.0.1:53032
I170517 07:32:32.436090 39305 gossip/client.go:136  [n9] closing client to node 1 (127.0.0.1:53032): received forward from node 1 to 2 (127.0.0.1:36501)
I170517 07:32:32.443788 39481 gossip/gossip.go:1210  [n9] node has connected to cluster via gossip
I170517 07:32:32.444612 39600 gossip/client.go:131  [n9] started gossip client to 127.0.0.1:36501
I170517 07:32:32.468078 39509 gossip/server.go:285  [n2] refusing gossip from node 9 (max 3 conns); forwarding to 6 ({tcp 127.0.0.1:42792})
I170517 07:32:32.468269 37982 gossip/gossip.go:297  [n9] NodeDescriptor set to node_id:9 address:<network_field:"tcp" address_field:"127.0.0.1:52709" > attrs:<> locality:<>
I170517 07:32:32.475603 39600 gossip/client.go:136  [n9] closing client to node 2 (127.0.0.1:36501): received forward from node 2 to 6 (127.0.0.1:42792)
I170517 07:32:32.477020 39319 gossip/client.go:131  [n9] started gossip client to 127.0.0.1:42792
W170517 07:32:32.539412 37982 gossip/gossip.go:1196  [n10] no incoming or outgoing connections
I170517 07:32:32.544372 39673 gossip/server.go:285  [n1] refusing gossip from node 10 (max 3 conns); forwarding to 4 ({tcp 127.0.0.1:56411})
I170517 07:32:32.545228 39661 gossip/client.go:131  [n10] started gossip client to 127.0.0.1:53032
I170517 07:32:32.555470 39661 gossip/client.go:136  [n10] closing client to node 1 (127.0.0.1:53032): received forward from node 1 to 4 (127.0.0.1:56411)
I170517 07:32:32.557481 39702 gossip/gossip.go:1210  [n10] node has connected to cluster via gossip
I170517 07:32:32.559541 39674 gossip/client.go:131  [n10] started gossip client to 127.0.0.1:56411
I170517 07:32:32.566152 37982 gossip/gossip.go:297  [n10] NodeDescriptor set to node_id:10 address:<network_field:"tcp" address_field:"127.0.0.1:37286" > attrs:<> locality:<>
I170517 07:32:32.620059 39677 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 07c8e96b at index 22
I170517 07:32:32.629558 39677 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 40, log entries: 12, rate-limit: 2.0 MiB/sec, 8ms
I170517 07:32:32.631020 39823 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 22 (id=07c8e96b, encoded size=8987, 1 rocksdb batches, 12 log entries)
I170517 07:32:32.646099 39823 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:32.652765 39677 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:32.657874 39863 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:32.683227 39907 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c9bd1006 at index 25
I170517 07:32:32.698615 39786 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:32.715497 39907 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 44, log entries: 15, rate-limit: 2.0 MiB/sec, 30ms
I170517 07:32:32.717629 39922 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 25 (id=c9bd1006, encoded size=11035, 1 rocksdb batches, 15 log entries)
I170517 07:32:32.719805 39922 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:32.749452 39907 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:32.769409 39837 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:32.811036 39988 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 613ba303 at index 29
I170517 07:32:32.814314 39988 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n4,s4):?: kv pairs: 49, log entries: 19, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:32.836015 39991 storage/replica_raftstorage.go:597  [s4,r1/?:{-}] applying preemptive snapshot at index 29 (id=613ba303, encoded size=13635, 1 rocksdb batches, 19 log entries)
I170517 07:32:32.838531 39991 storage/replica_raftstorage.go:605  [s4,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:32.914299 39988 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n4,s4):4): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:32:32.986372 40022 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n4,s4):4: [(n1,s1):1 (n2,s2):2 (n3,s3):3 (n4,s4):4]
I170517 07:32:33.018573 39984 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 7fa268d9 at index 35
I170517 07:32:33.036348 39984 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n5,s5):?: kv pairs: 56, log entries: 25, rate-limit: 2.0 MiB/sec, 17ms
I170517 07:32:33.037823 40001 storage/replica_raftstorage.go:597  [s5,r1/?:{-}] applying preemptive snapshot at index 35 (id=7fa268d9, encoded size=17275, 1 rocksdb batches, 25 log entries)
I170517 07:32:33.040569 40001 storage/replica_raftstorage.go:605  [s5,r1/?:/M{in-ax}] applied preemptive snapshot in 3ms [clear=0ms batch=0ms entries=2ms commit=0ms]
I170517 07:32:33.056302 39984 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n5,s5):5): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, (n4,s4):4, next=5]
I170517 07:32:33.094253 40026 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n5,s5):5: [(n1,s1):1 (n2,s2):2 (n3,s3):3 (n4,s4):4 (n5,s5):5]
I170517 07:32:33.161873 40119 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 4e07ca88 at index 37
I170517 07:32:33.167423 40119 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n6,s6):?: kv pairs: 59, log entries: 27, rate-limit: 2.0 MiB/sec, 5ms
I170517 07:32:33.172750 40030 storage/replica_raftstorage.go:597  [s6,r1/?:{-}] applying preemptive snapshot at index 37 (id=4e07ca88, encoded size=19027, 1 rocksdb batches, 27 log entries)
I170517 07:32:33.193193 40030 storage/replica_raftstorage.go:605  [s6,r1/?:/M{in-ax}] applied preemptive snapshot in 20ms [clear=6ms batch=0ms entries=8ms commit=0ms]
I170517 07:32:33.210706 40119 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n6,s6):6): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, (n4,s4):4, (n5,s5):5, next=6]
I170517 07:32:33.302377 40105 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n6,s6):6: [(n1,s1):1 (n2,s2):2 (n3,s3):3 (n4,s4):4 (n5,s5):5 (n6,s6):6]
I170517 07:32:33.346298 40135 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 780fe476 at index 41
I170517 07:32:33.406274 40135 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n7,s7):?: kv pairs: 64, log entries: 31, rate-limit: 2.0 MiB/sec, 59ms
I170517 07:32:33.429492 40065 storage/replica_raftstorage.go:597  [s7,r1/?:{-}] applying preemptive snapshot at index 41 (id=780fe476, encoded size=21819, 1 rocksdb batches, 31 log entries)
I170517 07:32:33.484031 40065 storage/replica_raftstorage.go:605  [s7,r1/?:/M{in-ax}] applied preemptive snapshot in 53ms [clear=15ms batch=0ms entries=19ms commit=0ms]
I170517 07:32:33.503431 40135 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n7,s7):7): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, (n4,s4):4, (n5,s5):5, (n6,s6):6, next=7]
I170517 07:32:33.625098 40252 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n7,s7):7: [(n1,s1):1 (n2,s2):2 (n3,s3):3 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7]
I170517 07:32:33.728078 40308 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot a8fcfeed at index 48
I170517 07:32:33.746733 40308 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n8,s8):?: kv pairs: 72, log entries: 38, rate-limit: 2.0 MiB/sec, 18ms
I170517 07:32:33.749367 40217 storage/replica_raftstorage.go:597  [s8,r1/?:{-}] applying preemptive snapshot at index 48 (id=a8fcfeed, encoded size=26139, 1 rocksdb batches, 38 log entries)
I170517 07:32:33.758006 40217 storage/replica_raftstorage.go:605  [s8,r1/?:/M{in-ax}] applied preemptive snapshot in 8ms [clear=0ms batch=0ms entries=7ms commit=0ms]
I170517 07:32:33.769679 40308 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n8,s8):8): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, next=8]
I170517 07:32:33.879931 40357 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n8,s8):8: [(n1,s1):1 (n2,s2):2 (n3,s3):3 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8]
I170517 07:32:34.020999 40374 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 59f45f4c at index 55
I170517 07:32:34.028272 40374 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n9,s9):?: kv pairs: 80, log entries: 45, rate-limit: 2.0 MiB/sec, 5ms
I170517 07:32:34.028493 40351 storage/replica_raftstorage.go:597  [s9,r1/?:{-}] applying preemptive snapshot at index 55 (id=59f45f4c, encoded size=30523, 1 rocksdb batches, 45 log entries)
I170517 07:32:34.038513 40351 storage/replica_raftstorage.go:605  [s9,r1/?:/M{in-ax}] applied preemptive snapshot in 10ms [clear=0ms batch=0ms entries=3ms commit=0ms]
I170517 07:32:34.080059 40374 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n9,s9):9): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, next=9]
I170517 07:32:34.345171 40401 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n9,s9):9: [(n1,s1):1 (n2,s2):2 (n3,s3):3 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9]
I170517 07:32:34.475070 40517 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot eb50eec7 at index 61
I170517 07:32:34.601294 40517 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n10,s10):?: kv pairs: 87, log entries: 51, rate-limit: 2.0 MiB/sec, 119ms
I170517 07:32:34.608140 40531 storage/replica_raftstorage.go:597  [s10,r1/?:{-}] applying preemptive snapshot at index 61 (id=eb50eec7, encoded size=34483, 1 rocksdb batches, 51 log entries)
I170517 07:32:34.688890 40531 storage/replica_raftstorage.go:605  [s10,r1/?:/M{in-ax}] applied preemptive snapshot in 81ms [clear=0ms batch=0ms entries=75ms commit=0ms]
I170517 07:32:34.695319 40517 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n10,s10):10): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, next=10]
I170517 07:32:34.964528 40628 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n10,s10):10: [(n1,s1):1 (n2,s2):2 (n3,s3):3 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9 (n10,s10):10]
I170517 07:32:35.339291 40714 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, (n10,s10):10, next=11]
I170517 07:32:35.533051 40809 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9]
I170517 07:32:35.684144 39788 storage/store.go:3152  [s3,r1/3:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:35.702497 38725 storage/store.go:2139  [replicaGC,s3,r1/3:/M{in-ax}] removing replica
I170517 07:32:35.704738 38725 storage/replica.go:684  [replicaGC,s3,r1/3:/M{in-ax}] removed 40 (32+8) keys in 2ms [clear=2ms commit=0ms]
I170517 07:32:35.781838 40720 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 1b593167 at index 82
I170517 07:32:35.813781 40720 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 110, log entries: 72, rate-limit: 2.0 MiB/sec, 27ms
I170517 07:32:35.841969 40829 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 82 (id=1b593167, encoded size=46861, 1 rocksdb batches, 72 log entries)
I170517 07:32:35.890210 40829 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 45ms [clear=0ms batch=0ms entries=29ms commit=11ms]
I170517 07:32:35.896503 40720 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):11): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, next=11]
I170517 07:32:36.236110 40965 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):11: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9 (n3,s3):11]
I170517 07:32:36.662772 41124 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):11): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, (n3,s3):11, next=12]
I170517 07:32:36.940283 41067 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):11: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9]
I170517 07:32:37.031881 39788 storage/store.go:3152  [s3,r1/11:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:37.099670 38725 storage/store.go:2139  [replicaGC,s3,r1/11:/M{in-ax}] removing replica
I170517 07:32:37.101533 38725 storage/replica.go:684  [replicaGC,s3,r1/11:/M{in-ax}] removed 40 (32+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:37.123490 41209 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot a477faaf at index 107
I170517 07:32:37.159343 41209 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 137, log entries: 97, rate-limit: 2.0 MiB/sec, 28ms
I170517 07:32:37.194210 41252 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 107 (id=a477faaf, encoded size=61189, 1 rocksdb batches, 97 log entries)
I170517 07:32:37.241027 41252 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 46ms [clear=0ms batch=0ms entries=30ms commit=1ms]
I170517 07:32:37.286881 41209 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):12): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, next=12]
I170517 07:32:37.386759 41299 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):12: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9 (n3,s3):12]
I170517 07:32:37.714584 41362 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):12): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, (n3,s3):12, next=13]
I170517 07:32:37.983920 41369 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):12: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9]
I170517 07:32:38.052278 41476 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 46b5ec82 at index 124
I170517 07:32:38.068883 41476 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 156, log entries: 114, rate-limit: 2.0 MiB/sec, 13ms
I170517 07:32:38.073782 39788 storage/store.go:3152  [s3,r1/12:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:38.098748 38725 storage/store.go:2139  [replicaGC,s3,r1/12:/M{in-ax}] removing replica
I170517 07:32:38.099656 38725 storage/replica.go:684  [replicaGC,s3,r1/12:/M{in-ax}] removed 40 (32+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:38.128111 41476 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):13): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, next=13]
I170517 07:32:38.378824 41558 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):13: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9 (n3,s3):13]
I170517 07:32:38.538521 38349 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot affa7185 at index 131
I170517 07:32:38.555311 38349 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):13: kv pairs: 163, log entries: 4, rate-limit: 8.0 MiB/sec, 16ms
I170517 07:32:38.557684 41489 storage/replica_raftstorage.go:597  [s3,r1/13:{-}] applying Raft snapshot at index 131 (id=affa7185, encoded size=17095, 1 rocksdb batches, 4 log entries)
I170517 07:32:38.559407 41489 storage/replica_raftstorage.go:605  [s3,r1/13:/M{in-ax}] applied Raft snapshot in 2ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:38.761587 41505 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):13): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, (n3,s3):13, next=14]
I170517 07:32:39.005092 41719 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):13: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9]
I170517 07:32:39.058075 39788 storage/store.go:3152  [s3,r1/13:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:39.063846 39788 storage/store.go:3152  [s3,r1/13:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:39.099052 38725 storage/store.go:2139  [replicaGC,s3,r1/13:/M{in-ax}] removing replica
I170517 07:32:39.099920 38725 storage/replica.go:684  [replicaGC,s3,r1/13:/M{in-ax}] removed 40 (32+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:39.155528 41762 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 6e2d0951 at index 144
I170517 07:32:39.181619 41762 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 177, log entries: 17, rate-limit: 2.0 MiB/sec, 25ms
I170517 07:32:39.188558 41765 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 144 (id=6e2d0951, encoded size=24489, 1 rocksdb batches, 17 log entries)
I170517 07:32:39.280223 41765 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 91ms [clear=0ms batch=0ms entries=24ms commit=66ms]
I170517 07:32:39.283625 41762 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):14): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, next=14]
I170517 07:32:39.459884 41823 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):14: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9 (n3,s3):14]
I170517 07:32:39.576144 41878 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):14): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, (n3,s3):14, next=15]
I170517 07:32:39.733979 41807 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):14: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9]
I170517 07:32:39.786031 39788 storage/store.go:3152  [s3,r1/14:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:39.791700 41931 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 0820bce2 at index 157
I170517 07:32:39.799549 41931 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 192, log entries: 30, rate-limit: 2.0 MiB/sec, 7ms
I170517 07:32:39.834538 38725 storage/store.go:2139  [replicaGC,s3,r1/14:/M{in-ax}] removing replica
I170517 07:32:39.835433 38725 storage/replica.go:684  [replicaGC,s3,r1/14:/M{in-ax}] removed 40 (32+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:39.841743 41931 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):15): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, next=15]
I170517 07:32:40.247226 42036 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):15: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9 (n3,s3):15]
I170517 07:32:40.353906 38349 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 30bc400a at index 165
I170517 07:32:40.396321 38349 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):15: kv pairs: 201, log entries: 38, rate-limit: 8.0 MiB/sec, 41ms
I170517 07:32:40.397750 42073 storage/replica_raftstorage.go:597  [s3,r1/15:{-}] applying Raft snapshot at index 165 (id=30bc400a, encoded size=38003, 1 rocksdb batches, 38 log entries)
I170517 07:32:40.426647 42073 storage/replica_raftstorage.go:605  [s3,r1/15:/M{in-ax}] applied Raft snapshot in 16ms [clear=0ms batch=0ms entries=14ms commit=0ms]
I170517 07:32:40.430624 42116 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):15): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, (n3,s3):15, next=16]
I170517 07:32:40.538098 42148 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):15: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9]
I170517 07:32:40.593206 39788 storage/store.go:3152  [s3,r1/15:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:40.634762 38725 storage/store.go:2139  [replicaGC,s3,r1/15:/M{in-ax}] removing replica
I170517 07:32:40.635517 42111 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 92c7f087 at index 173
I170517 07:32:40.640861 38725 storage/replica.go:684  [replicaGC,s3,r1/15:/M{in-ax}] removed 40 (32+8) keys in 6ms [clear=5ms commit=0ms]
I170517 07:32:40.762351 42111 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 210, log entries: 46, rate-limit: 2.0 MiB/sec, 97ms
I170517 07:32:40.764282 42144 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 173 (id=92c7f087, encoded size=42951, 1 rocksdb batches, 46 log entries)
I170517 07:32:40.772800 42144 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 8ms [clear=0ms batch=0ms entries=7ms commit=1ms]
I170517 07:32:40.787828 42111 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):16): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, next=16]
I170517 07:32:40.827634 42306 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):16: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9 (n3,s3):16]
I170517 07:32:41.000606 42265 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):16): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, (n3,s3):16, next=17]
I170517 07:32:41.087285 42319 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):16: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9]
I170517 07:32:41.100103 39788 storage/store.go:3152  [s3,r1/16:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:41.117557 42370 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 948a3934 at index 185
I170517 07:32:41.134360 42370 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 224, log entries: 58, rate-limit: 2.0 MiB/sec, 16ms
I170517 07:32:41.156082 38725 storage/store.go:2139  [replicaGC,s3,r1/16:/M{in-ax}] removing replica
I170517 07:32:41.157482 38725 storage/replica.go:684  [replicaGC,s3,r1/16:/M{in-ax}] removed 40 (32+8) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:41.165726 42370 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):17): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, next=17]
I170517 07:32:41.301786 42377 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):17: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9 (n3,s3):17]
I170517 07:32:41.343314 38349 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot ed8e39bb at index 190
I170517 07:32:41.368420 38349 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):17: kv pairs: 230, log entries: 63, rate-limit: 8.0 MiB/sec, 22ms
I170517 07:32:41.371961 42413 storage/replica_raftstorage.go:597  [s3,r1/17:{-}] applying Raft snapshot at index 190 (id=ed8e39bb, encoded size=54505, 1 rocksdb batches, 63 log entries)
I170517 07:32:41.500073 42413 storage/replica_raftstorage.go:605  [s3,r1/17:/M{in-ax}] applied Raft snapshot in 127ms [clear=0ms batch=0ms entries=122ms commit=2ms]
I170517 07:32:41.577031 42429 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):17): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, (n3,s3):17, next=18]
I170517 07:32:41.749436 42431 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):17: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9]
I170517 07:32:41.801463 38725 storage/store.go:2139  [replicaGC,s3,r1/17:/M{in-ax}] removing replica
I170517 07:32:41.802388 38725 storage/replica.go:684  [replicaGC,s3,r1/17:/M{in-ax}] removed 40 (32+8) keys in 0ms [clear=0ms commit=0ms]
W170517 07:32:41.813302 39914 storage/store.go:3160  [s1] got error from r1, replica (n3,s3):17: raft group deleted
I170517 07:32:41.852492 42539 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 876a09c9 at index 198
I170517 07:32:41.888127 42539 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 239, log entries: 71, rate-limit: 2.0 MiB/sec, 28ms
I170517 07:32:41.913947 42611 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 198 (id=876a09c9, encoded size=59457, 1 rocksdb batches, 71 log entries)
I170517 07:32:41.933807 42611 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 20ms [clear=0ms batch=0ms entries=18ms commit=1ms]
I170517 07:32:41.940546 42539 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):18): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, next=18]
I170517 07:32:41.957146 42556 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):18: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9 (n3,s3):18]
I170517 07:32:42.091203 42676 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):18): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, (n3,s3):18, next=19]
I170517 07:32:42.314863 42686 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):18: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9]
I170517 07:32:42.374959 39788 storage/store.go:3152  [s3,r1/18:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:42.378120 38725 storage/store.go:2139  [replicaGC,s3,r1/18:/M{in-ax}] removing replica
I170517 07:32:42.379131 38725 storage/replica.go:684  [replicaGC,s3,r1/18:/M{in-ax}] removed 40 (32+8) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:42.408889 42704 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 0a7e8a9a at index 210
I170517 07:32:42.440249 42704 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 253, log entries: 83, rate-limit: 2.0 MiB/sec, 22ms
I170517 07:32:42.460573 42727 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 210 (id=0a7e8a9a, encoded size=67465, 1 rocksdb batches, 83 log entries)
I170517 07:32:42.512806 42727 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 52ms [clear=0ms batch=0ms entries=43ms commit=7ms]
I170517 07:32:42.515424 42704 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):19): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, next=19]
I170517 07:32:42.547841 42753 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):19: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9 (n3,s3):19]
I170517 07:32:42.660095 42784 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):19): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, (n3,s3):19, next=20]
I170517 07:32:42.748326 42816 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):19: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9]
I170517 07:32:42.794060 39788 storage/store.go:3152  [s3,r1/19:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:42.827601 42825 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 431513b9 at index 220
I170517 07:32:42.847498 38725 storage/store.go:2139  [replicaGC,s3,r1/19:/M{in-ax}] removing replica
I170517 07:32:42.848541 38725 storage/replica.go:684  [replicaGC,s3,r1/19:/M{in-ax}] removed 40 (32+8) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:42.879492 42825 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 265, log entries: 93, rate-limit: 2.0 MiB/sec, 21ms
I170517 07:32:42.897193 42849 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 220 (id=431513b9, encoded size=74493, 1 rocksdb batches, 93 log entries)
I170517 07:32:42.911603 42849 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 14ms [clear=1ms batch=0ms entries=12ms commit=1ms]
I170517 07:32:42.930892 42825 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):20): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n10,s10):10, (n4,s4):4, (n5,s5):5, (n6,s6):6, (n7,s7):7, (n8,s8):8, (n9,s9):9, next=20]
I170517 07:32:43.057697 42862 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):20: [(n1,s1):1 (n2,s2):2 (n10,s10):10 (n4,s4):4 (n5,s5):5 (n6,s6):6 (n7,s7):7 (n8,s8):8 (n9,s9):9 (n3,s3):20]
W170517 07:32:43.129550 39790 storage/raft_transport.go:257  unable to accept Raft message from (n10,s10):?: no handler registered for (n1,s1):?
W170517 07:32:43.145898 39790 storage/raft_transport.go:257  unable to accept Raft message from (n5,s5):?: no handler registered for (n1,s1):?
W170517 07:32:43.146935 39790 storage/raft_transport.go:257  unable to accept Raft message from (n3,s3):20: no handler registered for (n1,s1):1
W170517 07:32:43.147361 39790 storage/raft_transport.go:257  unable to accept Raft message from (n8,s8):?: no handler registered for (n1,s1):?
W170517 07:32:43.147540 39790 storage/raft_transport.go:257  unable to accept Raft message from (n4,s4):?: no handler registered for (n1,s1):?
W170517 07:32:43.154101 39788 storage/store.go:3156  [s10] raft error: node 1 claims to not contain store 1 for replica (n1,s1):?: store 1 was not found
W170517 07:32:43.157684 39786 storage/raft_transport.go:442  raft transport stream to node 1 failed: store 1 was not found
I170517 07:32:43.166017 43012 storage/raft_transport.go:436  raft transport stream to node 1 established
W170517 07:32:43.167388 42953 storage/raft_transport.go:257  unable to accept Raft message from (n9,s9):?: no handler registered for (n1,s1):?
W170517 07:32:43.171286 43048 storage/store.go:3156  [s9] raft error: node 1 claims to not contain store 1 for replica (n1,s1):?: store 1 was not found
I170517 07:32:43.278103 39350 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:35620->127.0.0.1:41808: use of closed network connection
I170517 07:32:43.279430 39352 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:43.281943 39551 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:37286->127.0.0.1:35780: read: connection reset by peer
I170517 07:32:43.282054 38733 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:56411->127.0.0.1:36616: use of closed network connection
I170517 07:32:43.282637 38404 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:36501->127.0.0.1:52065: use of closed network connection
I170517 07:32:43.283153 38504 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:43.283479 38739 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:43.286350 38493 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:39790->127.0.0.1:54480: use of closed network connection
I170517 07:32:43.286759 38884 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:34960->127.0.0.1:54559: use of closed network connection
W170517 07:32:43.287458 39481 gossip/gossip.go:1196  [n9] no incoming or outgoing connections
I170517 07:32:43.288149 38742 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
I170517 07:32:43.291766 39342 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:52709->127.0.0.1:53613: use of closed network connection
I170517 07:32:43.292135 39155 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:41324->127.0.0.1:50194: use of closed network connection
I170517 07:32:43.292997 38989 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:42792->127.0.0.1:50654: use of closed network connection
I170517 07:32:43.293819 38341 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:53032->127.0.0.1:33889: use of closed network connection
I170517 07:32:43.302812 39355 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: write tcp 127.0.0.1:41856->127.0.0.1:35620: write: broken pipe"; Reconnecting to {127.0.0.1:35620 <nil>}
I170517 07:32:43.302965 39355 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestRaftRemoveRace (11.52s)
=== RUN   TestRemovePlaceholderRace
I170517 07:32:43.408067 43075 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:43.408423 43075 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:60829" > attrs:<> locality:<>
W170517 07:32:43.434322 43075 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:43.443486 43005 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:60829
I170517 07:32:43.459602 43075 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:45309" > attrs:<> locality:<>
W170517 07:32:43.506922 43075 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:43.508825 43018 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:60829
I170517 07:32:43.531621 43075 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:58433" > attrs:<> locality:<>
I170517 07:32:43.571424 43408 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c169475b at index 15
I170517 07:32:43.574363 43408 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:43.575634 43411 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=c169475b, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:43.577621 43411 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:43.583137 43408 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:43.597231 43427 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:43.615990 43431 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 3afcc742 at index 17
I170517 07:32:43.619758 43431 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:43.625724 43328 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=3afcc742, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:32:43.627252 43328 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:43.630393 43431 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:43.643422 43070 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:43.661340 43446 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:43.814907 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:32:43.824183 43355 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):2: [(n1,s1):1 (n3,s3):3]
I170517 07:32:43.833694 43274 storage/store.go:2139  [replicaGC,s2,r1/2:/M{in-ax}] removing replica
I170517 07:32:43.834317 43274 storage/replica.go:684  [replicaGC,s2,r1/2:/M{in-ax}] removed 33 (25+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:43.838439 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 99b876de at index 21
I170517 07:32:43.842091 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 42, log entries: 11, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:43.843135 43477 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 21 (id=99b876de, encoded size=10305, 1 rocksdb batches, 11 log entries)
I170517 07:32:43.844993 43477 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:43.848362 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):4): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=4]
W170517 07:32:43.863493 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):2: raft group deleted
I170517 07:32:43.867109 42956 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):4: [(n1,s1):1 (n3,s3):3 (n2,s2):4]
I170517 07:32:43.911223 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):4): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):4, next=5]
I170517 07:32:43.956887 42958 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):4: [(n1,s1):1 (n3,s3):3]
I170517 07:32:43.979394 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot a42aac67 at index 25
I170517 07:32:43.985179 43274 storage/store.go:2139  [replicaGC,s2,r1/4:/M{in-ax}] removing replica
I170517 07:32:43.985843 43274 storage/replica.go:684  [replicaGC,s2,r1/4:/M{in-ax}] removed 33 (25+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:44.024556 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 48, log entries: 15, rate-limit: 2.0 MiB/sec, 45ms
W170517 07:32:44.025116 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):4: raft group deleted
I170517 07:32:44.026055 43507 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 25 (id=a42aac67, encoded size=13489, 1 rocksdb batches, 15 log entries)
I170517 07:32:44.028098 43507 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:44.030692 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):5): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=5]
I170517 07:32:44.056591 43512 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):5: [(n1,s1):1 (n3,s3):3 (n2,s2):5]
I170517 07:32:44.069243 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):5): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):5, next=6]
I170517 07:32:44.077173 43514 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):5: [(n1,s1):1 (n3,s3):3]
I170517 07:32:44.091735 43444 storage/store.go:3152  [s2,r1/5:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:44.092060 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 82ab04a5 at index 29
I170517 07:32:44.094914 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 54, log entries: 19, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:44.096025 43274 storage/store.go:2139  [replicaGC,s2,r1/5:/M{in-ax}] removing replica
I170517 07:32:44.096723 43274 storage/replica.go:684  [replicaGC,s2,r1/5:/M{in-ax}] removed 33 (25+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:44.104338 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):6): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=6]
I170517 07:32:44.112963 43420 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):6: [(n1,s1):1 (n3,s3):3 (n2,s2):6]
I170517 07:32:44.121857 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):6): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):6, next=7]
I170517 07:32:44.135930 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot cbc407ef at index 31
I170517 07:32:44.140808 43422 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):6: [(n1,s1):1 (n3,s3):3]
I170517 07:32:44.141607 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):6: kv pairs: 57, log entries: 21, rate-limit: 8.0 MiB/sec, 4ms
I170517 07:32:44.150392 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c9ed2df7 at index 33
I170517 07:32:44.152614 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 08db48b3 at index 33
I170517 07:32:44.155157 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 9364ffed at index 33
I170517 07:32:44.157410 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot fd18fe92 at index 33
I170517 07:32:44.159299 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 6b571373 at index 33
I170517 07:32:44.161753 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot b222a00a at index 33
I170517 07:32:44.164199 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 3445167d at index 33
I170517 07:32:44.165430 43525 storage/replica_raftstorage.go:597  [s2,r1/6:{-}] applying Raft snapshot at index 31 (id=cbc407ef, encoded size=18293, 1 rocksdb batches, 21 log entries)
I170517 07:32:44.172909 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c91b5168 at index 34
I170517 07:32:44.175866 43525 storage/replica_raftstorage.go:605  [s2,r1/6:/M{in-ax}] applied Raft snapshot in 10ms [clear=0ms batch=0ms entries=2ms commit=8ms]
I170517 07:32:44.178158 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot fa8e9899 at index 34
I170517 07:32:44.182150 43444 storage/store.go:3152  [s2,r1/6:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:44.185025 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 61, log entries: 24, rate-limit: 2.0 MiB/sec, 6ms
I170517 07:32:44.185853 43498 storage/replica_raftstorage.go:597  [s2,r1/6:/M{in-ax}] applying preemptive snapshot at index 34 (id=fa8e9899, encoded size=20341, 1 rocksdb batches, 24 log entries)
I170517 07:32:44.188754 43498 storage/replica_raftstorage.go:605  [s2,r1/6:/M{in-ax}] applied preemptive snapshot in 3ms [clear=0ms batch=0ms entries=2ms commit=0ms]
I170517 07:32:44.189142 43274 storage/store.go:2139  [replicaGC,s2,r1/6:/M{in-ax}] removing replica
I170517 07:32:44.189847 43274 storage/replica.go:684  [replicaGC,s2,r1/6:/M{in-ax}] removed 33 (25+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:44.191780 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):7): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=7]
I170517 07:32:44.216464 43528 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):7: [(n1,s1):1 (n3,s3):3 (n2,s2):7]
I170517 07:32:44.227618 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):7): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):7, next=8]
I170517 07:32:44.235140 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 3f8d0ec8 at index 38
I170517 07:32:44.236625 43564 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):7: [(n1,s1):1 (n3,s3):3]
I170517 07:32:44.268621 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):7: kv pairs: 68, log entries: 28, rate-limit: 8.0 MiB/sec, 31ms
I170517 07:32:44.273148 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 596e1fa1 at index 39
I170517 07:32:44.282875 43607 storage/replica_raftstorage.go:597  [s2,r1/7:{-}] applying Raft snapshot at index 38 (id=3f8d0ec8, encoded size=23395, 1 rocksdb batches, 28 log entries)
I170517 07:32:44.284969 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 38f047b4 at index 39
I170517 07:32:44.287362 43607 storage/replica_raftstorage.go:605  [s2,r1/7:/M{in-ax}] applied Raft snapshot in 4ms [clear=0ms batch=0ms entries=3ms commit=0ms]
I170517 07:32:44.289826 43444 storage/store.go:3152  [s2,r1/7:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:44.292083 43274 storage/store.go:2139  [replicaGC,s2,r1/7:/M{in-ax}] removing replica
I170517 07:32:44.295987 43274 storage/replica.go:684  [replicaGC,s2,r1/7:/M{in-ax}] removed 34 (25+9) keys in 4ms [clear=3ms commit=0ms]
I170517 07:32:44.322009 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 89065771 at index 40
I170517 07:32:44.325215 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 69, log entries: 30, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:44.331951 43623 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 40 (id=89065771, encoded size=24501, 1 rocksdb batches, 30 log entries)
I170517 07:32:44.339416 43623 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 7ms [clear=0ms batch=0ms entries=6ms commit=0ms]
I170517 07:32:44.342031 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):8): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=8]
I170517 07:32:44.349991 43579 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):8: [(n1,s1):1 (n3,s3):3 (n2,s2):8]
I170517 07:32:44.369157 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):8): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):8, next=9]
I170517 07:32:44.380332 43585 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):8: [(n1,s1):1 (n3,s3):3]
I170517 07:32:44.389030 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot e08e67e1 at index 44
I170517 07:32:44.399142 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 75, log entries: 34, rate-limit: 2.0 MiB/sec, 9ms
I170517 07:32:44.399984 43444 storage/store.go:3152  [s2,r1/8:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:44.402571 43274 storage/store.go:2139  [replicaGC,s2,r1/8:/M{in-ax}] removing replica
I170517 07:32:44.403359 43274 storage/replica.go:684  [replicaGC,s2,r1/8:/M{in-ax}] removed 33 (25+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:44.404510 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):9): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=9]
I170517 07:32:44.415123 43703 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):9: [(n1,s1):1 (n3,s3):3 (n2,s2):9]
I170517 07:32:44.422994 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):9): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):9, next=10]
I170517 07:32:44.437764 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot ba7b1ac1 at index 47
I170517 07:32:44.441779 43656 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):9: [(n1,s1):1 (n3,s3):3]
I170517 07:32:44.443845 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):9: kv pairs: 81, log entries: 37, rate-limit: 8.0 MiB/sec, 5ms
I170517 07:32:44.445160 43714 storage/replica_raftstorage.go:597  [s2,r1/9:{-}] applying Raft snapshot at index 47 (id=ba7b1ac1, encoded size=30251, 1 rocksdb batches, 37 log entries)
I170517 07:32:44.449094 43714 storage/replica_raftstorage.go:605  [s2,r1/9:/M{in-ax}] applied Raft snapshot in 4ms [clear=0ms batch=0ms entries=3ms commit=0ms]
I170517 07:32:44.474831 43444 storage/store.go:3152  [s2,r1/9:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:44.476566 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot deab1823 at index 48
I170517 07:32:44.483679 43708 storage/replica_raftstorage.go:597  [s2,r1/9:/M{in-ax}] applying preemptive snapshot at index 48 (id=deab1823, encoded size=30869, 1 rocksdb batches, 38 log entries)
I170517 07:32:44.483788 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 81, log entries: 38, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:44.487746 43708 storage/replica_raftstorage.go:605  [s2,r1/9:/M{in-ax}] applied preemptive snapshot in 4ms [clear=0ms batch=0ms entries=3ms commit=0ms]
I170517 07:32:44.488213 43274 storage/store.go:2139  [replicaGC,s2,r1/9:/M{in-ax}] removing replica
I170517 07:32:44.488888 43274 storage/replica.go:684  [replicaGC,s2,r1/9:/M{in-ax}] removed 33 (25+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:44.492565 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):10): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=10]
I170517 07:32:44.505877 43696 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):10: [(n1,s1):1 (n3,s3):3 (n2,s2):10]
I170517 07:32:44.513813 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):10): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):10, next=11]
I170517 07:32:44.553879 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 7814583a at index 51
I170517 07:32:44.556677 43661 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):10: [(n1,s1):1 (n3,s3):3]
I170517 07:32:44.564614 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):10: kv pairs: 87, log entries: 41, rate-limit: 8.0 MiB/sec, 10ms
I170517 07:32:44.565876 43593 storage/replica_raftstorage.go:597  [s2,r1/10:{-}] applying Raft snapshot at index 51 (id=7814583a, encoded size=33435, 1 rocksdb batches, 41 log entries)
I170517 07:32:44.570217 43593 storage/replica_raftstorage.go:605  [s2,r1/10:/M{in-ax}] applied Raft snapshot in 4ms [clear=0ms batch=0ms entries=3ms commit=0ms]
I170517 07:32:44.585580 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 4f227c05 at index 52
I170517 07:32:44.592946 43444 storage/store.go:3152  [s2,r1/10:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:44.594151 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 87, log entries: 42, rate-limit: 2.0 MiB/sec, 8ms
I170517 07:32:44.599787 43274 storage/store.go:2139  [replicaGC,s2,r1/10:/M{in-ax}] removing replica
I170517 07:32:44.600855 43274 storage/replica.go:684  [replicaGC,s2,r1/10:/M{in-ax}] removed 33 (25+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:44.607725 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):11): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=11]
I170517 07:32:44.634654 43748 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):11: [(n1,s1):1 (n3,s3):3 (n2,s2):11]
I170517 07:32:44.643463 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):11): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):11, next=12]
I170517 07:32:44.649624 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 7a5e9f57 at index 54
I170517 07:32:44.653536 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):11: kv pairs: 90, log entries: 44, rate-limit: 8.0 MiB/sec, 3ms
I170517 07:32:44.654555 43741 storage/replica_raftstorage.go:597  [s2,r1/11:{-}] applying Raft snapshot at index 54 (id=7a5e9f57, encoded size=35677, 1 rocksdb batches, 44 log entries)
I170517 07:32:44.665864 43750 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):11: [(n1,s1):1 (n3,s3):3]
I170517 07:32:44.672096 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 5d0cf862 at index 56
I170517 07:32:44.676268 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 99e3766c at index 56
I170517 07:32:44.679737 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 3a8f73c1 at index 56
I170517 07:32:44.682651 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 647cbc00 at index 56
I170517 07:32:44.685141 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 94a9898d at index 56
I170517 07:32:44.686774 43741 storage/replica_raftstorage.go:605  [s2,r1/11:/M{in-ax}] applied Raft snapshot in 32ms [clear=0ms batch=0ms entries=31ms commit=0ms]
I170517 07:32:44.687108 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 571645a4 at index 56
I170517 07:32:44.691082 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 79a86960 at index 56
I170517 07:32:44.693792 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 6cc060ea at index 56
I170517 07:32:44.700341 43444 storage/store.go:3152  [s2,r1/11:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:44.717986 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 88f5917e at index 56
I170517 07:32:44.724123 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot be6eda63 at index 56
I170517 07:32:44.726677 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 3282af6c at index 56
I170517 07:32:44.730703 43274 storage/store.go:2139  [replicaGC,s2,r1/11:/M{in-ax}] removing replica
I170517 07:32:44.731852 43274 storage/replica.go:684  [replicaGC,s2,r1/11:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=0ms commit=1ms]
I170517 07:32:44.732679 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot affad3bd at index 56
I170517 07:32:44.739273 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 93, log entries: 46, rate-limit: 2.0 MiB/sec, 4ms
I170517 07:32:44.741901 43811 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 56 (id=affad3bd, encoded size=37237, 1 rocksdb batches, 46 log entries)
I170517 07:32:44.746267 43811 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 4ms [clear=0ms batch=0ms entries=3ms commit=0ms]
I170517 07:32:44.748969 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):12): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=12]
I170517 07:32:44.759256 43797 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):12: [(n1,s1):1 (n3,s3):3 (n2,s2):12]
I170517 07:32:44.830919 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):12): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):12, next=13]
I170517 07:32:44.845700 43858 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):12: [(n1,s1):1 (n3,s3):3]
I170517 07:32:44.854676 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot fd914530 at index 61
I170517 07:32:44.864417 43444 storage/store.go:3152  [s2,r1/12:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:44.865316 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 99, log entries: 51, rate-limit: 2.0 MiB/sec, 9ms
I170517 07:32:44.877123 43274 storage/store.go:2139  [replicaGC,s2,r1/12:/M{in-ax}] removing replica
I170517 07:32:44.878024 43274 storage/replica.go:684  [replicaGC,s2,r1/12:/M{in-ax}] removed 33 (25+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:44.881360 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):13): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=13]
I170517 07:32:44.904022 43814 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):13: [(n1,s1):1 (n3,s3):3 (n2,s2):13]
I170517 07:32:44.930899 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):13): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):13, next=14]
I170517 07:32:44.942220 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot ebdd5185 at index 65
I170517 07:32:44.951327 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):13: kv pairs: 104, log entries: 55, rate-limit: 8.0 MiB/sec, 9ms
I170517 07:32:44.952304 43838 storage/replica_raftstorage.go:597  [s2,r1/13:{-}] applying Raft snapshot at index 65 (id=ebdd5185, encoded size=43039, 1 rocksdb batches, 55 log entries)
I170517 07:32:44.957953 43838 storage/replica_raftstorage.go:605  [s2,r1/13:/M{in-ax}] applied Raft snapshot in 5ms [clear=0ms batch=0ms entries=4ms commit=0ms]
I170517 07:32:44.972010 43789 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):13: [(n1,s1):1 (n3,s3):3]
I170517 07:32:44.983768 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 7c2658d2 at index 67
I170517 07:32:44.991434 43444 storage/store.go:3152  [s2,r1/13:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:44.996708 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 107, log entries: 57, rate-limit: 2.0 MiB/sec, 12ms
I170517 07:32:44.998361 43274 storage/store.go:2139  [replicaGC,s2,r1/13:/M{in-ax}] removing replica
I170517 07:32:45.000299 43274 storage/replica.go:684  [replicaGC,s2,r1/13:/M{in-ax}] removed 33 (25+8) keys in 2ms [clear=0ms commit=1ms]
I170517 07:32:45.016895 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):14): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=14]
I170517 07:32:45.027116 43866 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):14: [(n1,s1):1 (n3,s3):3 (n2,s2):14]
I170517 07:32:45.045283 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):14): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):14, next=15]
I170517 07:32:45.075243 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 6358d0de at index 71
I170517 07:32:45.079252 43907 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):14: [(n1,s1):1 (n3,s3):3]
I170517 07:32:45.094446 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):14: kv pairs: 114, log entries: 61, rate-limit: 8.0 MiB/sec, 16ms
I170517 07:32:45.096034 43792 storage/replica_raftstorage.go:597  [s2,r1/14:{-}] applying Raft snapshot at index 71 (id=6358d0de, encoded size=47653, 1 rocksdb batches, 61 log entries)
I170517 07:32:45.104349 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 849b202f at index 72
I170517 07:32:45.106966 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c4592000 at index 72
I170517 07:32:45.107515 43792 storage/replica_raftstorage.go:605  [s2,r1/14:/M{in-ax}] applied Raft snapshot in 11ms [clear=6ms batch=0ms entries=4ms commit=1ms]
I170517 07:32:45.109423 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 08062a66 at index 72
I170517 07:32:45.109674 43444 storage/store.go:3152  [s2,r1/14:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:45.111565 43274 storage/store.go:2139  [replicaGC,s2,r1/14:/M{in-ax}] removing replica
I170517 07:32:45.112401 43274 storage/replica.go:684  [replicaGC,s2,r1/14:/M{in-ax}] removed 34 (25+9) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:45.131883 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 114, log entries: 62, rate-limit: 2.0 MiB/sec, 14ms
I170517 07:32:45.134427 43884 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 72 (id=08062a66, encoded size=48271, 1 rocksdb batches, 62 log entries)
I170517 07:32:45.140532 43884 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 6ms [clear=0ms batch=0ms entries=4ms commit=1ms]
I170517 07:32:45.144085 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):15): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=15]
I170517 07:32:45.162202 43898 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):15: [(n1,s1):1 (n3,s3):3 (n2,s2):15]
I170517 07:32:45.178223 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):15): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):15, next=16]
I170517 07:32:45.210921 43902 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):15: [(n1,s1):1 (n3,s3):3]
I170517 07:32:45.223373 43274 storage/store.go:2139  [replicaGC,s2,r1/15:/M{in-ax}] removing replica
I170517 07:32:45.224319 43274 storage/replica.go:684  [replicaGC,s2,r1/15:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:45.228683 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 595c1a4c at index 77
I170517 07:32:45.249020 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 120, log entries: 67, rate-limit: 2.0 MiB/sec, 18ms
I170517 07:32:45.252803 43803 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 77 (id=595c1a4c, encoded size=51472, 1 rocksdb batches, 67 log entries)
I170517 07:32:45.269589 43803 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 17ms [clear=0ms batch=0ms entries=15ms commit=1ms]
I170517 07:32:45.274563 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):16): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=16]
W170517 07:32:45.275960 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):15: raft group deleted
I170517 07:32:45.293421 43943 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):16: [(n1,s1):1 (n3,s3):3 (n2,s2):16]
I170517 07:32:45.321301 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):16): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):16, next=17]
I170517 07:32:45.335767 43970 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):16: [(n1,s1):1 (n3,s3):3]
I170517 07:32:45.349884 43274 storage/store.go:2139  [replicaGC,s2,r1/16:/M{in-ax}] removing replica
I170517 07:32:45.350337 43444 storage/store.go:3152  [s2,r1/16:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:45.350806 43274 storage/replica.go:684  [replicaGC,s2,r1/16:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:45.352141 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c15da561 at index 81
I170517 07:32:45.355869 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 126, log entries: 71, rate-limit: 2.0 MiB/sec, 3ms
W170517 07:32:45.370833 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):16: raft group deleted
I170517 07:32:45.372527 43950 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 81 (id=c15da561, encoded size=54657, 1 rocksdb batches, 71 log entries)
I170517 07:32:45.385444 43950 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 13ms [clear=0ms batch=0ms entries=11ms commit=1ms]
I170517 07:32:45.390277 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):17): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=17]
I170517 07:32:45.401448 43889 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):17: [(n1,s1):1 (n3,s3):3 (n2,s2):17]
I170517 07:32:45.414109 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):17): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):17, next=18]
I170517 07:32:45.436357 43919 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):17: [(n1,s1):1 (n3,s3):3]
I170517 07:32:45.447442 43444 storage/store.go:3152  [s2,r1/17:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:45.448240 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot ca85bb12 at index 85
I170517 07:32:45.449583 43274 storage/store.go:2139  [replicaGC,s2,r1/17:/M{in-ax}] removing replica
I170517 07:32:45.450960 43274 storage/replica.go:684  [replicaGC,s2,r1/17:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:45.454881 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 132, log entries: 75, rate-limit: 2.0 MiB/sec, 5ms
I170517 07:32:45.467892 43777 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 85 (id=ca85bb12, encoded size=57841, 1 rocksdb batches, 75 log entries)
I170517 07:32:45.482907 43777 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 15ms [clear=0ms batch=0ms entries=13ms commit=1ms]
I170517 07:32:45.505082 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):18): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=18]
I170517 07:32:45.518212 44034 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):18: [(n1,s1):1 (n3,s3):3 (n2,s2):18]
I170517 07:32:45.527677 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):18): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):18, next=19]
I170517 07:32:45.537304 43996 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):18: [(n1,s1):1 (n3,s3):3]
I170517 07:32:45.545808 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 8e26b3f1 at index 89
I170517 07:32:45.553669 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 138, log entries: 79, rate-limit: 2.0 MiB/sec, 7ms
I170517 07:32:45.556616 43274 storage/store.go:2139  [replicaGC,s2,r1/18:/M{in-ax}] removing replica
I170517 07:32:45.558556 43444 storage/store.go:3152  [s2,r1/18:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:45.560221 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):19): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=19]
I170517 07:32:45.564496 43274 storage/replica.go:684  [replicaGC,s2,r1/18:/M{in-ax}] removed 33 (25+8) keys in 7ms [clear=0ms commit=7ms]
I170517 07:32:45.573686 43998 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):19: [(n1,s1):1 (n3,s3):3 (n2,s2):19]
I170517 07:32:45.613043 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):19): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):19, next=20]
I170517 07:32:45.641299 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 9947faea at index 92
I170517 07:32:45.648948 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):19: kv pairs: 142, log entries: 82, rate-limit: 8.0 MiB/sec, 7ms
I170517 07:32:45.651068 44043 storage/replica_raftstorage.go:597  [s2,r1/19:{-}] applying Raft snapshot at index 92 (id=9947faea, encoded size=63133, 1 rocksdb batches, 82 log entries)
I170517 07:32:45.667089 44032 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):19: [(n1,s1):1 (n3,s3):3]
I170517 07:32:45.672259 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 5da59a42 at index 94
I170517 07:32:45.676688 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot adeb79fc at index 94
I170517 07:32:45.678104 44043 storage/replica_raftstorage.go:605  [s2,r1/19:/M{in-ax}] applied Raft snapshot in 27ms [clear=0ms batch=0ms entries=22ms commit=1ms]
I170517 07:32:45.680361 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot a9138756 at index 94
I170517 07:32:45.681288 43444 storage/store.go:3152  [s2,r1/19:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:45.687015 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 145, log entries: 84, rate-limit: 2.0 MiB/sec, 5ms
I170517 07:32:45.695232 44069 storage/replica_raftstorage.go:597  [s2,r1/19:/M{in-ax}] applying preemptive snapshot at index 94 (id=a9138756, encoded size=64693, 1 rocksdb batches, 84 log entries)
I170517 07:32:45.742694 43444 storage/store.go:3152  [s2,r1/19:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:45.752753 44069 storage/replica_raftstorage.go:605  [s2,r1/19:/M{in-ax}] applied preemptive snapshot in 57ms [clear=0ms batch=0ms entries=52ms commit=4ms]
I170517 07:32:45.753236 43274 storage/store.go:2139  [replicaGC,s2,r1/19:/M{in-ax}] removing replica
I170517 07:32:45.754281 43274 storage/replica.go:684  [replicaGC,s2,r1/19:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:45.764609 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):20): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=20]
I170517 07:32:45.773736 44095 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):20: [(n1,s1):1 (n3,s3):3 (n2,s2):20]
I170517 07:32:45.788811 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):20): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):20, next=21]
I170517 07:32:45.813741 43983 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):20: [(n1,s1):1 (n3,s3):3]
I170517 07:32:45.817296 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 067ff4fd at index 99
I170517 07:32:45.822018 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 3223cf45 at index 100
I170517 07:32:45.833561 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 55058237 at index 100
I170517 07:32:45.842841 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):20: kv pairs: 153, log entries: 89, rate-limit: 8.0 MiB/sec, 25ms
I170517 07:32:45.844605 44132 storage/replica_raftstorage.go:597  [s2,r1/20:{-}] applying Raft snapshot at index 99 (id=067ff4fd, encoded size=68239, 1 rocksdb batches, 89 log entries)
I170517 07:32:45.851567 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot dc259bb3 at index 100
I170517 07:32:45.855688 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot dcb6cb45 at index 100
I170517 07:32:45.860777 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot df6a83d6 at index 100
I170517 07:32:45.865520 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot b9948700 at index 100
I170517 07:32:45.877765 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 77fcc1c1 at index 100
I170517 07:32:45.881489 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 7c6b752b at index 100
I170517 07:32:45.882172 44132 storage/replica_raftstorage.go:605  [s2,r1/20:/M{in-ax}] applied Raft snapshot in 37ms [clear=0ms batch=0ms entries=34ms commit=2ms]
I170517 07:32:45.888747 43444 storage/store.go:3152  [s2,r1/20:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:45.911804 43444 storage/store.go:3152  [s2,r1/20:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:45.933435 44101 storage/replica_raftstorage.go:597  [s2,r1/20:/M{in-ax}] applying preemptive snapshot at index 100 (id=7c6b752b, encoded size=68857, 1 rocksdb batches, 90 log entries)
I170517 07:32:45.952087 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 153, log entries: 90, rate-limit: 2.0 MiB/sec, 50ms
I170517 07:32:45.952807 44101 storage/replica_raftstorage.go:605  [s2,r1/20:/M{in-ax}] applied preemptive snapshot in 19ms [clear=2ms batch=0ms entries=11ms commit=1ms]
I170517 07:32:45.953482 43274 storage/store.go:2139  [replicaGC,s2,r1/20:/M{in-ax}] removing replica
I170517 07:32:45.955038 43274 storage/replica.go:684  [replicaGC,s2,r1/20:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:45.955820 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):21): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=21]
I170517 07:32:45.968674 44104 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):21: [(n1,s1):1 (n3,s3):3 (n2,s2):21]
I170517 07:32:45.994588 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):21): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):21, next=22]
I170517 07:32:46.006674 44195 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):21: [(n1,s1):1 (n3,s3):3]
I170517 07:32:46.014124 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 2058d47f at index 103
I170517 07:32:46.018137 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 0ac9b937 at index 104
I170517 07:32:46.024153 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 159, log entries: 94, rate-limit: 2.0 MiB/sec, 5ms
I170517 07:32:46.027322 44183 storage/replica_raftstorage.go:597  [s2,r1/21:{-}] applying preemptive snapshot at index 104 (id=0ac9b937, encoded size=72041, 1 rocksdb batches, 94 log entries)
I170517 07:32:46.049176 44183 storage/replica_raftstorage.go:605  [s2,r1/21:/M{in-ax}] applied preemptive snapshot in 22ms [clear=0ms batch=0ms entries=13ms commit=5ms]
I170517 07:32:46.058049 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):21: kv pairs: 159, log entries: 93, rate-limit: 8.0 MiB/sec, 36ms
I170517 07:32:46.061948 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):22): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=22]
I170517 07:32:46.068119 43444 storage/store.go:3152  [s2,r1/21:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:46.072891 44227 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):22: [(n1,s1):1 (n3,s3):3 (n2,s2):22]
I170517 07:32:46.085964 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):22): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):22, next=23]
I170517 07:32:46.089399 43274 storage/store.go:2139  [replicaGC,s2,r1/21:/M{in-ax}] removing replica
I170517 07:32:46.090992 43274 storage/replica.go:684  [replicaGC,s2,r1/21:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=0ms commit=0ms]
E170517 07:32:46.115330 43444 storage/store.go:3150  [s2,r1/22:{-}] unable to add to replica GC queue: replica not initialized
I170517 07:32:46.115439 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 0889dd4e at index 108
I170517 07:32:46.116696 44242 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):22: [(n1,s1):1 (n3,s3):3]
I170517 07:32:46.126768 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):22: kv pairs: 165, log entries: 3, rate-limit: 8.0 MiB/sec, 7ms
I170517 07:32:46.133403 44244 storage/replica_raftstorage.go:597  [s2,r1/22:{-}] applying Raft snapshot at index 108 (id=0889dd4e, encoded size=15021, 1 rocksdb batches, 3 log entries)
I170517 07:32:46.133519 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot f74c788e at index 109
I170517 07:32:46.136043 44244 storage/replica_raftstorage.go:605  [s2,r1/22:/M{in-ax}] applied Raft snapshot in 2ms [clear=0ms batch=0ms entries=0ms commit=1ms]
I170517 07:32:46.143267 43444 storage/store.go:3152  [s2,r1/22:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:46.149488 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 165, log entries: 4, rate-limit: 2.0 MiB/sec, 12ms
I170517 07:32:46.151256 44156 storage/replica_raftstorage.go:597  [s2,r1/22:/M{in-ax}] applying preemptive snapshot at index 109 (id=f74c788e, encoded size=15639, 1 rocksdb batches, 4 log entries)
I170517 07:32:46.153496 44156 storage/replica_raftstorage.go:605  [s2,r1/22:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:46.154599 43274 storage/store.go:2139  [replicaGC,s2,r1/22:/M{in-ax}] removing replica
I170517 07:32:46.155448 43274 storage/replica.go:684  [replicaGC,s2,r1/22:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:46.158192 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):23): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=23]
W170517 07:32:46.186110 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):22: raft group deleted
I170517 07:32:46.199185 44161 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):23: [(n1,s1):1 (n3,s3):3 (n2,s2):23]
I170517 07:32:46.215284 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):23): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):23, next=24]
I170517 07:32:46.224110 44203 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):23: [(n1,s1):1 (n3,s3):3]
I170517 07:32:46.226027 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot bf50b2e9 at index 112
I170517 07:32:46.231709 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):23: kv pairs: 171, log entries: 7, rate-limit: 8.0 MiB/sec, 5ms
I170517 07:32:46.233002 44261 storage/replica_raftstorage.go:597  [s2,r1/23:{-}] applying Raft snapshot at index 112 (id=bf50b2e9, encoded size=18205, 1 rocksdb batches, 7 log entries)
I170517 07:32:46.241826 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 2a9d2fa8 at index 113
I170517 07:32:46.244293 44261 storage/replica_raftstorage.go:605  [s2,r1/23:/M{in-ax}] applied Raft snapshot in 11ms [clear=0ms batch=0ms entries=1ms commit=6ms]
I170517 07:32:46.247562 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c9dc1a2c at index 113
I170517 07:32:46.251102 43444 storage/store.go:3152  [s2,r1/23:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:46.253211 43274 storage/store.go:2139  [replicaGC,s2,r1/23:/M{in-ax}] removing replica
I170517 07:32:46.254268 43274 storage/replica.go:684  [replicaGC,s2,r1/23:/M{in-ax}] removed 34 (25+9) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:46.255174 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 886c4559 at index 113
I170517 07:32:46.261950 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 171, log entries: 8, rate-limit: 2.0 MiB/sec, 6ms
W170517 07:32:46.263952 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):23: raft group deleted
I170517 07:32:46.264158 44122 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 113 (id=886c4559, encoded size=18823, 1 rocksdb batches, 8 log entries)
I170517 07:32:46.266406 44122 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:46.269285 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):24): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=24]
I170517 07:32:46.276886 44278 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):24: [(n1,s1):1 (n3,s3):3 (n2,s2):24]
I170517 07:32:46.290122 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):24): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):24, next=25]
I170517 07:32:46.324772 44175 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):24: [(n1,s1):1 (n3,s3):3]
I170517 07:32:46.350089 43444 storage/store.go:3152  [s2,r1/24:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:46.353461 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot a7e1f63f at index 118
I170517 07:32:46.391063 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 178, log entries: 13, rate-limit: 2.0 MiB/sec, 36ms
I170517 07:32:46.392928 43274 storage/store.go:2139  [replicaGC,s2,r1/24:/M{in-ax}] removing replica
I170517 07:32:46.394132 43274 storage/replica.go:684  [replicaGC,s2,r1/24:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:46.395104 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):25): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=25]
I170517 07:32:46.415771 44251 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):25: [(n1,s1):1 (n3,s3):3 (n2,s2):25]
I170517 07:32:46.432061 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):25): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):25, next=26]
I170517 07:32:46.435197 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot ac7c5941 at index 122
I170517 07:32:46.443169 44282 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):25: [(n1,s1):1 (n3,s3):3]
I170517 07:32:46.453324 44256 storage/replica_raftstorage.go:597  [s2,r1/25:{-}] applying Raft snapshot at index 122 (id=ac7c5941, encoded size=25095, 1 rocksdb batches, 17 log entries)
I170517 07:32:46.455072 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 87d941e6 at index 124
I170517 07:32:46.456423 44256 storage/replica_raftstorage.go:605  [s2,r1/25:/M{in-ax}] applied Raft snapshot in 3ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:46.461671 43444 storage/store.go:3152  [s2,r1/25:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:46.465682 43274 storage/store.go:2139  [replicaGC,s2,r1/25:/M{in-ax}] removing replica
I170517 07:32:46.466117 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):25: kv pairs: 183, log entries: 17, rate-limit: 8.0 MiB/sec, 30ms
I170517 07:32:46.466891 43274 storage/replica.go:684  [replicaGC,s2,r1/25:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:46.479058 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 186, log entries: 19, rate-limit: 2.0 MiB/sec, 12ms
I170517 07:32:46.481317 44338 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 124 (id=87d941e6, encoded size=26655, 1 rocksdb batches, 19 log entries)
I170517 07:32:46.484488 44338 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 3ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:46.487331 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):26): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=26]
I170517 07:32:46.501403 44319 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):26: [(n1,s1):1 (n3,s3):3 (n2,s2):26]
I170517 07:32:46.527888 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):26): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):26, next=27]
I170517 07:32:46.539952 44360 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):26: [(n1,s1):1 (n3,s3):3]
I170517 07:32:46.579524 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 9992e109 at index 128
I170517 07:32:46.580774 43444 storage/store.go:3152  [s2,r1/26:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:46.586622 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 192, log entries: 23, rate-limit: 2.0 MiB/sec, 6ms
I170517 07:32:46.587818 43274 storage/store.go:2139  [replicaGC,s2,r1/26:/M{in-ax}] removing replica
I170517 07:32:46.588965 43274 storage/replica.go:684  [replicaGC,s2,r1/26:/M{in-ax}] removed 34 (25+9) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:46.592071 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):27): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=27]
I170517 07:32:46.601562 44402 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):27: [(n1,s1):1 (n3,s3):3 (n2,s2):27]
I170517 07:32:46.622782 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):27): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):27, next=28]
I170517 07:32:46.631611 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot bfba73fa at index 130
I170517 07:32:46.639177 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):27: kv pairs: 195, log entries: 25, rate-limit: 8.0 MiB/sec, 4ms
I170517 07:32:46.639289 44349 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):27: [(n1,s1):1 (n3,s3):3]
I170517 07:32:46.642491 44347 storage/replica_raftstorage.go:597  [s2,r1/27:{-}] applying Raft snapshot at index 130 (id=bfba73fa, encoded size=31466, 1 rocksdb batches, 25 log entries)
I170517 07:32:46.646621 44347 storage/replica_raftstorage.go:605  [s2,r1/27:/M{in-ax}] applied Raft snapshot in 4ms [clear=0ms batch=0ms entries=2ms commit=1ms]
I170517 07:32:46.664211 43444 storage/store.go:3152  [s2,r1/27:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:46.664337 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 3920be3b at index 132
I170517 07:32:46.672996 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 198, log entries: 27, rate-limit: 2.0 MiB/sec, 8ms
I170517 07:32:46.673781 44406 storage/replica_raftstorage.go:597  [s2,r1/27:/M{in-ax}] applying preemptive snapshot at index 132 (id=3920be3b, encoded size=33028, 1 rocksdb batches, 27 log entries)
I170517 07:32:46.677887 44406 storage/replica_raftstorage.go:605  [s2,r1/27:/M{in-ax}] applied preemptive snapshot in 4ms [clear=0ms batch=0ms entries=2ms commit=1ms]
I170517 07:32:46.678414 43274 storage/store.go:2139  [replicaGC,s2,r1/27:/M{in-ax}] removing replica
I170517 07:32:46.679338 43274 storage/replica.go:684  [replicaGC,s2,r1/27:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:46.682583 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):28): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=28]
I170517 07:32:46.694501 44287 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):28: [(n1,s1):1 (n3,s3):3 (n2,s2):28]
I170517 07:32:46.718623 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):28): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):28, next=29]
I170517 07:32:46.727941 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot ab6b6479 at index 135
I170517 07:32:46.729396 44423 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):28: [(n1,s1):1 (n3,s3):3]
I170517 07:32:46.732161 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):28: kv pairs: 204, log entries: 30, rate-limit: 8.0 MiB/sec, 4ms
I170517 07:32:46.735074 44333 storage/replica_raftstorage.go:597  [s2,r1/28:{-}] applying Raft snapshot at index 135 (id=ab6b6479, encoded size=35597, 1 rocksdb batches, 30 log entries)
I170517 07:32:46.746428 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 608dc83c at index 136
I170517 07:32:46.749435 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 60994379 at index 136
I170517 07:32:46.751596 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot a3625e83 at index 136
I170517 07:32:46.754046 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 8b09c570 at index 136
I170517 07:32:46.754918 44333 storage/replica_raftstorage.go:605  [s2,r1/28:/M{in-ax}] applied Raft snapshot in 20ms [clear=0ms batch=0ms entries=4ms commit=1ms]
I170517 07:32:46.759334 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 6f8d1327 at index 136
I170517 07:32:46.760112 43444 storage/store.go:3152  [s2,r1/28:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:46.767724 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 204, log entries: 31, rate-limit: 2.0 MiB/sec, 8ms
I170517 07:32:46.769003 44453 storage/replica_raftstorage.go:597  [s2,r1/28:/M{in-ax}] applying preemptive snapshot at index 136 (id=6f8d1327, encoded size=36216, 1 rocksdb batches, 31 log entries)
I170517 07:32:46.781731 44453 storage/replica_raftstorage.go:605  [s2,r1/28:/M{in-ax}] applied preemptive snapshot in 13ms [clear=1ms batch=0ms entries=2ms commit=1ms]
I170517 07:32:46.782175 43274 storage/store.go:2139  [replicaGC,s2,r1/28:/M{in-ax}] removing replica
I170517 07:32:46.783404 43274 storage/replica.go:684  [replicaGC,s2,r1/28:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:46.797969 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):29): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=29]
I170517 07:32:46.805807 44445 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):29: [(n1,s1):1 (n3,s3):3 (n2,s2):29]
I170517 07:32:46.821967 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):29): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):29, next=30]
I170517 07:32:46.835069 44472 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):29: [(n1,s1):1 (n3,s3):3]
I170517 07:32:46.848614 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 009498f5 at index 139
I170517 07:32:46.854942 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):29: kv pairs: 210, log entries: 34, rate-limit: 8.0 MiB/sec, 5ms
I170517 07:32:46.857364 44482 storage/replica_raftstorage.go:597  [s2,r1/29:{-}] applying Raft snapshot at index 139 (id=009498f5, encoded size=38785, 1 rocksdb batches, 34 log entries)
I170517 07:32:46.866356 44482 storage/replica_raftstorage.go:605  [s2,r1/29:/M{in-ax}] applied Raft snapshot in 9ms [clear=2ms batch=0ms entries=6ms commit=1ms]
I170517 07:32:46.871679 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 05e7ed6b at index 140
I170517 07:32:46.874699 43444 storage/store.go:3152  [s2,r1/29:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:46.876734 43274 storage/store.go:2139  [replicaGC,s2,r1/29:/M{in-ax}] removing replica
I170517 07:32:46.877899 43274 storage/replica.go:684  [replicaGC,s2,r1/29:/M{in-ax}] removed 34 (25+9) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:46.884167 44428 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 140 (id=05e7ed6b, encoded size=39404, 1 rocksdb batches, 35 log entries)
I170517 07:32:46.888378 44428 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 4ms [clear=0ms batch=0ms entries=2ms commit=1ms]
I170517 07:32:46.889326 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 210, log entries: 35, rate-limit: 2.0 MiB/sec, 17ms
I170517 07:32:46.891578 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):30): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=30]
I170517 07:32:46.898549 44380 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):30: [(n1,s1):1 (n3,s3):3 (n2,s2):30]
I170517 07:32:46.907873 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):30): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):30, next=31]
I170517 07:32:46.911541 43444 storage/store.go:3152  [s2,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:46.923701 44490 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):30: [(n1,s1):1 (n3,s3):3]
I170517 07:32:46.931181 43444 storage/store.go:3152  [s2,r1/30:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:46.933144 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 24cd46af at index 144
I170517 07:32:46.939783 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 9ac60502 at index 144
I170517 07:32:46.961419 43274 storage/store.go:2139  [replicaGC,s2,r1/30:/M{in-ax}] removing replica
I170517 07:32:46.962875 43274 storage/replica.go:684  [replicaGC,s2,r1/30:/M{in-ax}] removed 34 (25+9) keys in 1ms [clear=1ms commit=0ms]
W170517 07:32:46.968958 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):30: raft group deleted
I170517 07:32:46.970845 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 216, log entries: 39, rate-limit: 2.0 MiB/sec, 30ms
I170517 07:32:46.973474 44382 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 144 (id=9ac60502, encoded size=42595, 1 rocksdb batches, 39 log entries)
I170517 07:32:46.977833 44382 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 4ms [clear=0ms batch=0ms entries=3ms commit=1ms]
I170517 07:32:46.981467 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):31): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=31]
I170517 07:32:46.992421 44531 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):31: [(n1,s1):1 (n3,s3):3 (n2,s2):31]
I170517 07:32:47.000336 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):31): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):31, next=32]
I170517 07:32:47.014432 44562 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):31: [(n1,s1):1 (n3,s3):3]
I170517 07:32:47.024674 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c048561f at index 148
I170517 07:32:47.043474 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 222, log entries: 43, rate-limit: 2.0 MiB/sec, 13ms
I170517 07:32:47.045420 43444 storage/store.go:3152  [s2,r1/31:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:47.061560 43274 storage/store.go:2139  [replicaGC,s2,r1/31:/M{in-ax}] removing replica
I170517 07:32:47.066877 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):32): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=32]
I170517 07:32:47.067095 43274 storage/replica.go:684  [replicaGC,s2,r1/31:/M{in-ax}] removed 33 (25+8) keys in 4ms [clear=1ms commit=3ms]
I170517 07:32:47.090524 44579 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):32: [(n1,s1):1 (n3,s3):3 (n2,s2):32]
I170517 07:32:47.099916 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):32): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):32, next=33]
I170517 07:32:47.111279 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 38532e29 at index 152
I170517 07:32:47.138595 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):32: kv pairs: 227, log entries: 47, rate-limit: 8.0 MiB/sec, 26ms
I170517 07:32:47.140589 44520 storage/replica_raftstorage.go:597  [s2,r1/32:{-}] applying Raft snapshot at index 152 (id=38532e29, encoded size=48391, 1 rocksdb batches, 47 log entries)
I170517 07:32:47.149670 44568 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):32: [(n1,s1):1 (n3,s3):3]
I170517 07:32:47.156901 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 4b517197 at index 154
I170517 07:32:47.190338 44520 storage/replica_raftstorage.go:605  [s2,r1/32:/M{in-ax}] applied Raft snapshot in 39ms [clear=0ms batch=0ms entries=37ms commit=1ms]
I170517 07:32:47.196368 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 8170ffb6 at index 155
I170517 07:32:47.201116 43444 storage/store.go:3152  [s2,r1/32:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:47.208349 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 231, log entries: 50, rate-limit: 2.0 MiB/sec, 11ms
I170517 07:32:47.212619 43444 storage/store.go:3152  [s2,r1/32:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:47.213347 44527 storage/replica_raftstorage.go:597  [s2,r1/32:/M{in-ax}] applying preemptive snapshot at index 155 (id=8170ffb6, encoded size=50445, 1 rocksdb batches, 50 log entries)
I170517 07:32:47.220931 44527 storage/replica_raftstorage.go:605  [s2,r1/32:/M{in-ax}] applied preemptive snapshot in 7ms [clear=1ms batch=0ms entries=5ms commit=1ms]
I170517 07:32:47.221855 43274 storage/store.go:2139  [replicaGC,s2,r1/32:/M{in-ax}] removing replica
I170517 07:32:47.223222 43274 storage/replica.go:684  [replicaGC,s2,r1/32:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:47.224092 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):33): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=33]
I170517 07:32:47.238065 44496 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):33: [(n1,s1):1 (n3,s3):3 (n2,s2):33]
I170517 07:32:47.265493 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):33): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):33, next=34]
I170517 07:32:47.278705 44612 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):33: [(n1,s1):1 (n3,s3):3]
I170517 07:32:47.280425 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot d0effd49 at index 158
I170517 07:32:47.286414 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):33: kv pairs: 237, log entries: 53, rate-limit: 8.0 MiB/sec, 5ms
I170517 07:32:47.287857 44627 storage/replica_raftstorage.go:597  [s2,r1/33:{-}] applying Raft snapshot at index 158 (id=d0effd49, encoded size=53013, 1 rocksdb batches, 53 log entries)
I170517 07:32:47.300056 44627 storage/replica_raftstorage.go:605  [s2,r1/33:/M{in-ax}] applied Raft snapshot in 9ms [clear=1ms batch=0ms entries=7ms commit=1ms]
I170517 07:32:47.303883 43444 storage/store.go:3152  [s2,r1/33:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:47.304045 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 8b804101 at index 159
I170517 07:32:47.308979 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 237, log entries: 54, rate-limit: 2.0 MiB/sec, 4ms
I170517 07:32:47.311116 44643 storage/replica_raftstorage.go:597  [s2,r1/33:/M{in-ax}] applying preemptive snapshot at index 159 (id=8b804101, encoded size=53633, 1 rocksdb batches, 54 log entries)
I170517 07:32:47.343112 44643 storage/replica_raftstorage.go:605  [s2,r1/33:/M{in-ax}] applied preemptive snapshot in 32ms [clear=1ms batch=0ms entries=25ms commit=5ms]
I170517 07:32:47.347631 43274 storage/store.go:2139  [replicaGC,s2,r1/33:/M{in-ax}] removing replica
I170517 07:32:47.348862 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):34): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=34]
I170517 07:32:47.349040 43274 storage/replica.go:684  [replicaGC,s2,r1/33:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:47.355915 44629 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):34: [(n1,s1):1 (n3,s3):3 (n2,s2):34]
I170517 07:32:47.368389 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):34): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):34, next=35]
I170517 07:32:47.382301 44633 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):34: [(n1,s1):1 (n3,s3):3]
I170517 07:32:47.387263 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 740eecf5 at index 162
I170517 07:32:47.396075 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 06b5255d at index 163
I170517 07:32:47.412520 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):34: kv pairs: 243, log entries: 57, rate-limit: 8.0 MiB/sec, 24ms
I170517 07:32:47.412823 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c2630cf3 at index 163
I170517 07:32:47.414003 44661 storage/replica_raftstorage.go:597  [s2,r1/34:{-}] applying Raft snapshot at index 162 (id=740eecf5, encoded size=56205, 1 rocksdb batches, 57 log entries)
I170517 07:32:47.415702 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 1fe3ce73 at index 163
I170517 07:32:47.421365 44661 storage/replica_raftstorage.go:605  [s2,r1/34:/M{in-ax}] applied Raft snapshot in 7ms [clear=1ms batch=0ms entries=5ms commit=1ms]
I170517 07:32:47.435027 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 02559a8d at index 163
I170517 07:32:47.436425 43444 storage/store.go:3152  [s2,r1/34:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:47.442982 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 243, log entries: 58, rate-limit: 2.0 MiB/sec, 6ms
I170517 07:32:47.444697 44653 storage/replica_raftstorage.go:597  [s2,r1/34:/M{in-ax}] applying preemptive snapshot at index 163 (id=02559a8d, encoded size=56825, 1 rocksdb batches, 58 log entries)
I170517 07:32:47.453786 44653 storage/replica_raftstorage.go:605  [s2,r1/34:/M{in-ax}] applied preemptive snapshot in 9ms [clear=1ms batch=0ms entries=4ms commit=3ms]
I170517 07:32:47.454498 43274 storage/store.go:2139  [replicaGC,s2,r1/34:/M{in-ax}] removing replica
I170517 07:32:47.455999 43274 storage/replica.go:684  [replicaGC,s2,r1/34:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:47.456581 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):35): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=35]
I170517 07:32:47.466509 44513 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):35: [(n1,s1):1 (n3,s3):3 (n2,s2):35]
I170517 07:32:47.474140 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):35): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):35, next=36]
I170517 07:32:47.483940 44691 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):35: [(n1,s1):1 (n3,s3):3]
I170517 07:32:47.499117 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot a98070eb at index 168
E170517 07:32:47.507489 43444 storage/store.go:3150  [s2,r1/35:{-}] unable to add to replica GC queue: replica not initialized
I170517 07:32:47.514130 44677 storage/replica_raftstorage.go:597  [s2,r1/35:{-}] applying preemptive snapshot at index 168 (id=a98070eb, encoded size=60039, 1 rocksdb batches, 63 log entries)
I170517 07:32:47.528818 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 249, log entries: 63, rate-limit: 2.0 MiB/sec, 29ms
I170517 07:32:47.531552 44677 storage/replica_raftstorage.go:605  [s2,r1/35:/M{in-ax}] applied preemptive snapshot in 17ms [clear=0ms batch=0ms entries=15ms commit=1ms]
I170517 07:32:47.534550 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):36): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=36]
I170517 07:32:47.541945 44602 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):36: [(n1,s1):1 (n3,s3):3 (n2,s2):36]
I170517 07:32:47.553229 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):36): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):36, next=37]
I170517 07:32:47.571195 44606 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):36: [(n1,s1):1 (n3,s3):3]
I170517 07:32:47.598071 43444 storage/store.go:3152  [s2,r1/36:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:47.599605 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 38f487ba at index 172
I170517 07:32:47.600979 43274 storage/store.go:2139  [replicaGC,s2,r1/36:/M{in-ax}] removing replica
I170517 07:32:47.602283 43274 storage/replica.go:684  [replicaGC,s2,r1/36:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:47.610827 44712 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 172 (id=38f487ba, encoded size=63231, 1 rocksdb batches, 67 log entries)
I170517 07:32:47.619518 44712 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 9ms [clear=1ms batch=0ms entries=6ms commit=1ms]
I170517 07:32:47.620684 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 255, log entries: 67, rate-limit: 2.0 MiB/sec, 18ms
I170517 07:32:47.627497 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):37): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=37]
I170517 07:32:47.636099 44683 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):37: [(n1,s1):1 (n3,s3):3 (n2,s2):37]
I170517 07:32:47.645133 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):37): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):37, next=38]
I170517 07:32:47.653515 44754 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):37: [(n1,s1):1 (n3,s3):3]
I170517 07:32:47.658832 43444 storage/store.go:3152  [s2,r1/37:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:47.662110 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 0e1e38b2 at index 176
I170517 07:32:47.664721 43274 storage/store.go:2139  [replicaGC,s2,r1/37:/M{in-ax}] removing replica
I170517 07:32:47.666055 43274 storage/replica.go:684  [replicaGC,s2,r1/37:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:47.671356 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 261, log entries: 71, rate-limit: 2.0 MiB/sec, 9ms
I170517 07:32:47.677701 44704 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 176 (id=0e1e38b2, encoded size=66423, 1 rocksdb batches, 71 log entries)
I170517 07:32:47.684631 44704 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 7ms [clear=0ms batch=0ms entries=5ms commit=1ms]
I170517 07:32:47.687849 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):38): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=38]
I170517 07:32:47.700756 44728 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):38: [(n1,s1):1 (n3,s3):3 (n2,s2):38]
I170517 07:32:47.712806 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):38): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):38, next=39]
I170517 07:32:47.722897 44788 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):38: [(n1,s1):1 (n3,s3):3]
I170517 07:32:47.729076 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot a23f5b65 at index 180
I170517 07:32:47.737561 43444 storage/store.go:3152  [s2,r1/38:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:47.739738 43274 storage/store.go:2139  [replicaGC,s2,r1/38:/M{in-ax}] removing replica
I170517 07:32:47.741161 43274 storage/replica.go:684  [replicaGC,s2,r1/38:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:47.759274 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 267, log entries: 75, rate-limit: 2.0 MiB/sec, 12ms
W170517 07:32:47.765045 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):38: raft group deleted
I170517 07:32:47.765151 44756 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 180 (id=a23f5b65, encoded size=69615, 1 rocksdb batches, 75 log entries)
I170517 07:32:47.772369 44756 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 7ms [clear=0ms batch=0ms entries=5ms commit=1ms]
I170517 07:32:47.775010 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):39): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=39]
I170517 07:32:47.793187 44796 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):39: [(n1,s1):1 (n3,s3):3 (n2,s2):39]
I170517 07:32:47.805746 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):39): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):39, next=40]
I170517 07:32:47.821196 44746 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):39: [(n1,s1):1 (n3,s3):3]
I170517 07:32:47.827600 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot d52e5bda at index 186
I170517 07:32:47.835566 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 275, log entries: 81, rate-limit: 2.0 MiB/sec, 7ms
I170517 07:32:47.846212 43444 storage/store.go:3152  [s2,r1/39:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:47.851916 44760 storage/replica_raftstorage.go:597  [s2,r1/39:/M{in-ax}] applying preemptive snapshot at index 186 (id=d52e5bda, encoded size=73787, 1 rocksdb batches, 81 log entries)
I170517 07:32:47.917081 44760 storage/replica_raftstorage.go:605  [s2,r1/39:/M{in-ax}] applied preemptive snapshot in 48ms [clear=19ms batch=0ms entries=27ms commit=1ms]
I170517 07:32:47.917587 43274 storage/store.go:2139  [replicaGC,s2,r1/39:/M{in-ax}] removing replica
I170517 07:32:47.919366 43274 storage/replica.go:684  [replicaGC,s2,r1/39:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:47.922269 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):40): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=40]
I170517 07:32:47.938647 44762 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):40: [(n1,s1):1 (n3,s3):3 (n2,s2):40]
I170517 07:32:47.950284 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):40): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):40, next=41]
I170517 07:32:47.980978 44825 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):40: [(n1,s1):1 (n3,s3):3]
I170517 07:32:47.984999 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 397ea6dd at index 190
I170517 07:32:47.993603 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):40: kv pairs: 282, log entries: 85, rate-limit: 8.0 MiB/sec, 8ms
I170517 07:32:48.004011 44807 storage/replica_raftstorage.go:597  [s2,r1/40:{-}] applying Raft snapshot at index 190 (id=397ea6dd, encoded size=76849, 1 rocksdb batches, 85 log entries)
I170517 07:32:48.010990 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot cb598ffe at index 191
I170517 07:32:48.015637 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 18733833 at index 191
I170517 07:32:48.020166 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 81fba501 at index 191
I170517 07:32:48.023844 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c93dd6e4 at index 191
I170517 07:32:48.027932 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c208bc7d at index 191
I170517 07:32:48.030501 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 0f129631 at index 191
I170517 07:32:48.035487 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot d7fd0e2d at index 191
I170517 07:32:48.044842 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot dca56832 at index 191
I170517 07:32:48.047827 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 6449b2a7 at index 191
I170517 07:32:48.050466 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 3ccca72a at index 191
I170517 07:32:48.053792 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot e80b8ddb at index 191
I170517 07:32:48.056819 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 44c44c2a at index 191
I170517 07:32:48.061004 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot f1063dd3 at index 191
I170517 07:32:48.062211 44807 storage/replica_raftstorage.go:605  [s2,r1/40:/M{in-ax}] applied Raft snapshot in 42ms [clear=1ms batch=0ms entries=30ms commit=2ms]
I170517 07:32:48.071671 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot d22499ec at index 191
I170517 07:32:48.082034 43444 storage/store.go:3152  [s2,r1/40:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:48.083051 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot be51ad86 at index 191
I170517 07:32:48.085244 43274 storage/store.go:2139  [replicaGC,s2,r1/40:/M{in-ax}] removing replica
I170517 07:32:48.086749 43274 storage/replica.go:684  [replicaGC,s2,r1/40:/M{in-ax}] removed 34 (25+9) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:48.092722 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 282, log entries: 86, rate-limit: 2.0 MiB/sec, 8ms
I170517 07:32:48.098708 44875 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 191 (id=be51ad86, encoded size=77469, 1 rocksdb batches, 86 log entries)
I170517 07:32:48.119678 44875 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 21ms [clear=1ms batch=0ms entries=18ms commit=1ms]
I170517 07:32:48.122765 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):41): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=41]
I170517 07:32:48.135386 44913 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):41: [(n1,s1):1 (n3,s3):3 (n2,s2):41]
I170517 07:32:48.145578 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):41): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):41, next=42]
I170517 07:32:48.163582 44947 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):41: [(n1,s1):1 (n3,s3):3]
I170517 07:32:48.187382 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot fc2a8fc6 at index 195
I170517 07:32:48.190540 43444 storage/store.go:3152  [s2,r1/41:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:48.195152 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 288, log entries: 90, rate-limit: 2.0 MiB/sec, 6ms
I170517 07:32:48.196745 43274 storage/store.go:2139  [replicaGC,s2,r1/41:/M{in-ax}] removing replica
I170517 07:32:48.198239 43274 storage/replica.go:684  [replicaGC,s2,r1/41:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:48.202852 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):42): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=42]
I170517 07:32:48.226305 44895 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):42: [(n1,s1):1 (n3,s3):3 (n2,s2):42]
I170517 07:32:48.280447 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):42): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):42, next=43]
I170517 07:32:48.280945 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot a1b9e60b at index 198
I170517 07:32:48.305970 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):42: kv pairs: 291, log entries: 3, rate-limit: 8.0 MiB/sec, 24ms
I170517 07:32:48.309735 44880 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):42: [(n1,s1):1 (n3,s3):3]
I170517 07:32:48.311929 44938 storage/replica_raftstorage.go:597  [s2,r1/42:{-}] applying Raft snapshot at index 198 (id=a1b9e60b, encoded size=21937, 1 rocksdb batches, 3 log entries)
I170517 07:32:48.320973 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot f8924706 at index 200
I170517 07:32:48.326518 44938 storage/replica_raftstorage.go:605  [s2,r1/42:/M{in-ax}] applied Raft snapshot in 11ms [clear=1ms batch=0ms entries=5ms commit=1ms]
I170517 07:32:48.328391 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 5392570a at index 200
I170517 07:32:48.328567 43444 storage/store.go:3152  [s2,r1/42:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:48.335939 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 294, log entries: 5, rate-limit: 2.0 MiB/sec, 7ms
I170517 07:32:48.336781 44860 storage/replica_raftstorage.go:597  [s2,r1/42:/M{in-ax}] applying preemptive snapshot at index 200 (id=5392570a, encoded size=23501, 1 rocksdb batches, 5 log entries)
I170517 07:32:48.340555 44860 storage/replica_raftstorage.go:605  [s2,r1/42:/M{in-ax}] applied preemptive snapshot in 4ms [clear=1ms batch=0ms entries=0ms commit=1ms]
I170517 07:32:48.341102 43274 storage/store.go:2139  [replicaGC,s2,r1/42:/M{in-ax}] removing replica
I170517 07:32:48.345392 43274 storage/replica.go:684  [replicaGC,s2,r1/42:/M{in-ax}] removed 33 (25+8) keys in 4ms [clear=3ms commit=0ms]
I170517 07:32:48.350248 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):43): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=43]
W170517 07:32:48.362179 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):42: raft group deleted
I170517 07:32:48.365271 44862 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):43: [(n1,s1):1 (n3,s3):3 (n2,s2):43]
I170517 07:32:48.392033 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):43): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):43, next=44]
I170517 07:32:48.418418 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot d07d9033 at index 202
I170517 07:32:48.426754 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):43: kv pairs: 297, log entries: 7, rate-limit: 8.0 MiB/sec, 8ms
I170517 07:32:48.428112 44864 storage/replica_raftstorage.go:597  [s2,r1/43:{-}] applying Raft snapshot at index 202 (id=d07d9033, encoded size=25129, 1 rocksdb batches, 7 log entries)
I170517 07:32:48.431295 44864 storage/replica_raftstorage.go:605  [s2,r1/43:/M{in-ax}] applied Raft snapshot in 3ms [clear=1ms batch=0ms entries=1ms commit=1ms]
I170517 07:32:48.450753 44980 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):43: [(n1,s1):1 (n3,s3):3]
I170517 07:32:48.477244 43444 storage/store.go:3152  [s2,r1/43:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:48.478212 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 4e96d3c4 at index 204
I170517 07:32:48.479236 43274 storage/store.go:2139  [replicaGC,s2,r1/43:/M{in-ax}] removing replica
I170517 07:32:48.480717 43274 storage/replica.go:684  [replicaGC,s2,r1/43:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:48.481174 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 44790fd5 at index 204
I170517 07:32:48.501763 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 300, log entries: 9, rate-limit: 2.0 MiB/sec, 20ms
I170517 07:32:48.504806 45012 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 204 (id=44790fd5, encoded size=26693, 1 rocksdb batches, 9 log entries)
I170517 07:32:48.511804 43444 storage/store.go:3152  [s2,r1/?:{-}] added to replica GC queue (peer suggestion)
I170517 07:32:48.512797 45012 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 8ms [clear=1ms batch=0ms entries=1ms commit=1ms]
I170517 07:32:48.513741 43274 storage/store.go:2139  [replicaGC,s2,r1/?:/M{in-ax}] removing replica
I170517 07:32:48.515497 43274 storage/replica.go:684  [replicaGC,s2,r1/?:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=1ms]
I170517 07:32:48.519146 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):44): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=44]
I170517 07:32:48.535592 45001 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):44: [(n1,s1):1 (n3,s3):3 (n2,s2):44]
I170517 07:32:48.550097 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):44): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):44, next=45]
I170517 07:32:48.572189 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot be902c0a at index 208
I170517 07:32:48.578433 45058 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):44: [(n1,s1):1 (n3,s3):3]
I170517 07:32:48.587502 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 12e5d032 at index 210
I170517 07:32:48.591454 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):44: kv pairs: 305, log entries: 13, rate-limit: 8.0 MiB/sec, 12ms
I170517 07:32:48.591552 44971 storage/replica_raftstorage.go:597  [s2,r1/44:{-}] applying Raft snapshot at index 208 (id=be902c0a, encoded size=29301, 1 rocksdb batches, 13 log entries)
I170517 07:32:48.596909 44971 storage/replica_raftstorage.go:605  [s2,r1/44:/M{in-ax}] applied Raft snapshot in 5ms [clear=1ms batch=0ms entries=3ms commit=1ms]
I170517 07:32:48.600545 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 4171c839 at index 210
I170517 07:32:48.604199 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot e469103d at index 210
I170517 07:32:48.623226 43444 storage/store.go:3152  [s2,r1/44:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:48.627695 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 308, log entries: 15, rate-limit: 2.0 MiB/sec, 23ms
I170517 07:32:48.629202 45055 storage/replica_raftstorage.go:597  [s2,r1/44:/M{in-ax}] applying preemptive snapshot at index 210 (id=e469103d, encoded size=30865, 1 rocksdb batches, 15 log entries)
I170517 07:32:48.634736 45055 storage/replica_raftstorage.go:605  [s2,r1/44:/M{in-ax}] applied preemptive snapshot in 4ms [clear=1ms batch=0ms entries=1ms commit=1ms]
I170517 07:32:48.635127 43274 storage/store.go:2139  [replicaGC,s2,r1/44:/M{in-ax}] removing replica
I170517 07:32:48.636672 43274 storage/replica.go:684  [replicaGC,s2,r1/44:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:48.639443 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):45): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=45]
I170517 07:32:48.658116 44977 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):45: [(n1,s1):1 (n3,s3):3 (n2,s2):45]
I170517 07:32:48.683751 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):45): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):45, next=46]
I170517 07:32:48.701981 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 65d87b8b at index 215
I170517 07:32:48.704075 45097 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):45: [(n1,s1):1 (n3,s3):3]
I170517 07:32:48.714594 45035 storage/replica_raftstorage.go:597  [s2,r1/45:{-}] applying Raft snapshot at index 215 (id=65d87b8b, encoded size=33941, 1 rocksdb batches, 20 log entries)
I170517 07:32:48.718934 45035 storage/replica_raftstorage.go:605  [s2,r1/45:/M{in-ax}] applied Raft snapshot in 4ms [clear=1ms batch=0ms entries=2ms commit=1ms]
I170517 07:32:48.723068 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot f72bc5ee at index 216
I170517 07:32:48.731347 43444 storage/store.go:3152  [s2,r1/45:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:48.733244 43274 storage/store.go:2139  [replicaGC,s2,r1/45:/M{in-ax}] removing replica
I170517 07:32:48.735026 43274 storage/replica.go:684  [replicaGC,s2,r1/45:/M{in-ax}] removed 34 (25+9) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:48.735931 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):45: kv pairs: 315, log entries: 20, rate-limit: 8.0 MiB/sec, 33ms
I170517 07:32:48.743604 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 315, log entries: 21, rate-limit: 2.0 MiB/sec, 7ms
I170517 07:32:48.747065 45078 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 216 (id=f72bc5ee, encoded size=34561, 1 rocksdb batches, 21 log entries)
I170517 07:32:48.750575 45078 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 3ms [clear=1ms batch=0ms entries=1ms commit=1ms]
I170517 07:32:48.753339 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):46): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=46]
I170517 07:32:48.770118 45040 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):46: [(n1,s1):1 (n3,s3):3 (n2,s2):46]
I170517 07:32:48.777784 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):46): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):46, next=47]
I170517 07:32:48.821579 45079 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):46: [(n1,s1):1 (n3,s3):3]
I170517 07:32:48.827665 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 28046db5 at index 220
I170517 07:32:48.831757 43444 storage/store.go:3152  [s2,r1/46:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:48.835002 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 321, log entries: 25, rate-limit: 2.0 MiB/sec, 6ms
I170517 07:32:48.836461 43274 storage/store.go:2139  [replicaGC,s2,r1/46:/M{in-ax}] removing replica
I170517 07:32:48.839492 43274 storage/replica.go:684  [replicaGC,s2,r1/46:/M{in-ax}] removed 33 (25+8) keys in 3ms [clear=2ms commit=0ms]
I170517 07:32:48.840915 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):47): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=47]
I170517 07:32:48.852260 45020 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):47: [(n1,s1):1 (n3,s3):3 (n2,s2):47]
W170517 07:32:48.862366 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):46: raft group deleted
I170517 07:32:48.865914 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):47): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):47, next=48]
I170517 07:32:48.874193 45114 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):47: [(n1,s1):1 (n3,s3):3]
I170517 07:32:48.883356 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot e6791d4d at index 224
I170517 07:32:48.892726 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 327, log entries: 29, rate-limit: 2.0 MiB/sec, 9ms
I170517 07:32:48.895121 45170 storage/replica_raftstorage.go:597  [s2,r1/47:{-}] applying preemptive snapshot at index 224 (id=e6791d4d, encoded size=40941, 1 rocksdb batches, 29 log entries)
I170517 07:32:48.901063 45170 storage/replica_raftstorage.go:605  [s2,r1/47:/M{in-ax}] applied preemptive snapshot in 6ms [clear=1ms batch=0ms entries=4ms commit=1ms]
I170517 07:32:48.904051 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):48): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=48]
I170517 07:32:48.915372 45158 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):48: [(n1,s1):1 (n3,s3):3 (n2,s2):48]
I170517 07:32:48.927056 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):48): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):48, next=49]
I170517 07:32:48.935787 45144 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):48: [(n1,s1):1 (n3,s3):3]
I170517 07:32:48.959023 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot a48245e5 at index 228
I170517 07:32:48.979000 43444 storage/store.go:3152  [s2,r1/48:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:48.986549 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 333, log entries: 33, rate-limit: 2.0 MiB/sec, 27ms
I170517 07:32:48.987922 43274 storage/store.go:2139  [replicaGC,s2,r1/48:/M{in-ax}] removing replica
I170517 07:32:48.989412 43274 storage/replica.go:684  [replicaGC,s2,r1/48:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:48.995654 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):49): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=49]
I170517 07:32:49.003693 45064 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):49: [(n1,s1):1 (n3,s3):3 (n2,s2):49]
I170517 07:32:49.018003 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):49): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):49, next=50]
I170517 07:32:49.027618 45208 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):49: [(n1,s1):1 (n3,s3):3]
I170517 07:32:49.042549 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot f89c35f5 at index 232
E170517 07:32:49.050122 43444 storage/store.go:3150  [s2,r1/49:{-}] unable to add to replica GC queue: replica not initialized
I170517 07:32:49.065650 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 339, log entries: 37, rate-limit: 2.0 MiB/sec, 18ms
I170517 07:32:49.068294 45214 storage/replica_raftstorage.go:597  [s2,r1/49:{-}] applying preemptive snapshot at index 232 (id=f89c35f5, encoded size=47325, 1 rocksdb batches, 37 log entries)
I170517 07:32:49.074947 45214 storage/replica_raftstorage.go:605  [s2,r1/49:/M{in-ax}] applied preemptive snapshot in 6ms [clear=1ms batch=0ms entries=4ms commit=1ms]
I170517 07:32:49.078093 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):50): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=50]
I170517 07:32:49.091115 45166 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):50: [(n1,s1):1 (n3,s3):3 (n2,s2):50]
I170517 07:32:49.130235 43444 storage/store.go:3152  [s2,r1/49:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:49.136176 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):50): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):50, next=51]
I170517 07:32:49.149145 45174 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):50: [(n1,s1):1 (n3,s3):3]
I170517 07:32:49.158734 43444 storage/store.go:3152  [s2,r1/50:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:49.159902 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 926a8a1c at index 236
I170517 07:32:49.170233 43274 storage/store.go:2139  [replicaGC,s2,r1/50:/M{in-ax}] removing replica
I170517 07:32:49.172633 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 38568a75 at index 236
I170517 07:32:49.175694 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 989f5d43 at index 236
I170517 07:32:49.179184 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 6eadcb57 at index 236
I170517 07:32:49.185617 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot ff2dbc41 at index 236
I170517 07:32:49.189427 43274 storage/replica.go:684  [replicaGC,s2,r1/50:/M{in-ax}] removed 34 (25+9) keys in 19ms [clear=17ms commit=2ms]
I170517 07:32:49.204872 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 345, log entries: 41, rate-limit: 2.0 MiB/sec, 18ms
I170517 07:32:49.208939 45268 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 236 (id=ff2dbc41, encoded size=50517, 1 rocksdb batches, 41 log entries)
I170517 07:32:49.217869 45268 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 9ms [clear=1ms batch=0ms entries=6ms commit=1ms]
I170517 07:32:49.220369 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):51): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=51]
I170517 07:32:49.237239 45270 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):51: [(n1,s1):1 (n3,s3):3 (n2,s2):51]
I170517 07:32:49.246153 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):51): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):51, next=52]
I170517 07:32:49.266027 45151 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):51: [(n1,s1):1 (n3,s3):3]
I170517 07:32:49.276710 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot fe16431f at index 242
I170517 07:32:49.278932 43444 storage/store.go:3152  [s2,r1/51:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:49.286480 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 353, log entries: 47, rate-limit: 2.0 MiB/sec, 9ms
I170517 07:32:49.288290 45153 storage/replica_raftstorage.go:597  [s2,r1/51:/M{in-ax}] applying preemptive snapshot at index 242 (id=fe16431f, encoded size=54689, 1 rocksdb batches, 47 log entries)
I170517 07:32:49.306682 45153 storage/replica_raftstorage.go:605  [s2,r1/51:/M{in-ax}] applied preemptive snapshot in 18ms [clear=1ms batch=0ms entries=15ms commit=1ms]
I170517 07:32:49.312360 43274 storage/store.go:2139  [replicaGC,s2,r1/51:/M{in-ax}] removing replica
I170517 07:32:49.314017 43274 storage/replica.go:684  [replicaGC,s2,r1/51:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:49.317440 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):52): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=52]
I170517 07:32:49.329461 45288 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):52: [(n1,s1):1 (n3,s3):3 (n2,s2):52]
I170517 07:32:49.342306 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):52): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):52, next=53]
I170517 07:32:49.389292 45332 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):52: [(n1,s1):1 (n3,s3):3]
I170517 07:32:49.392981 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot d81b740b at index 246
I170517 07:32:49.398473 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot fb3ad75a at index 247
I170517 07:32:49.401283 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 8d2f5f64 at index 247
I170517 07:32:49.421341 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):52: kv pairs: 360, log entries: 51, rate-limit: 8.0 MiB/sec, 13ms
I170517 07:32:49.421449 45296 storage/replica_raftstorage.go:597  [s2,r1/52:{-}] applying Raft snapshot at index 246 (id=d81b740b, encoded size=57751, 1 rocksdb batches, 51 log entries)
I170517 07:32:49.428455 45296 storage/replica_raftstorage.go:605  [s2,r1/52:/M{in-ax}] applied Raft snapshot in 7ms [clear=1ms batch=0ms entries=4ms commit=1ms]
I170517 07:32:49.431054 43444 storage/store.go:3152  [s2,r1/52:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:49.432916 43274 storage/store.go:2139  [replicaGC,s2,r1/52:/M{in-ax}] removing replica
I170517 07:32:49.434464 43274 storage/replica.go:684  [replicaGC,s2,r1/52:/M{in-ax}] removed 34 (25+9) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:49.444806 45349 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 247 (id=8d2f5f64, encoded size=58371, 1 rocksdb batches, 52 log entries)
I170517 07:32:49.464465 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 360, log entries: 52, rate-limit: 2.0 MiB/sec, 28ms
I170517 07:32:49.484263 45349 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 29ms [clear=2ms batch=0ms entries=25ms commit=1ms]
I170517 07:32:49.487157 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):53): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=53]
I170517 07:32:49.498483 45336 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):53: [(n1,s1):1 (n3,s3):3 (n2,s2):53]
I170517 07:32:49.511706 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):53): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):53, next=54]
I170517 07:32:49.521024 45363 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):53: [(n1,s1):1 (n3,s3):3]
I170517 07:32:49.533222 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot ca4e8dfd at index 251
I170517 07:32:49.548456 43444 storage/store.go:3152  [s2,r1/53:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:49.565486 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 366, log entries: 56, rate-limit: 2.0 MiB/sec, 32ms
I170517 07:32:49.574520 43274 storage/store.go:2139  [replicaGC,s2,r1/53:/M{in-ax}] removing replica
I170517 07:32:49.576538 43274 storage/replica.go:684  [replicaGC,s2,r1/53:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:49.577940 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):54): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=54]
I170517 07:32:49.587909 45227 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):54: [(n1,s1):1 (n3,s3):3 (n2,s2):54]
I170517 07:32:49.599093 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):54): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):54, next=55]
I170517 07:32:49.635371 45382 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):54: [(n1,s1):1 (n3,s3):3]
E170517 07:32:49.656426 43444 storage/store.go:3150  [s2,r1/54:{-}] unable to add to replica GC queue: replica not initialized
I170517 07:32:49.657717 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot f365b5b7 at index 255
I170517 07:32:49.665787 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 372, log entries: 60, rate-limit: 2.0 MiB/sec, 5ms
I170517 07:32:49.669379 45411 storage/replica_raftstorage.go:597  [s2,r1/54:{-}] applying preemptive snapshot at index 255 (id=f365b5b7, encoded size=64755, 1 rocksdb batches, 60 log entries)
I170517 07:32:49.678889 45411 storage/replica_raftstorage.go:605  [s2,r1/54:/M{in-ax}] applied preemptive snapshot in 9ms [clear=1ms batch=0ms entries=7ms commit=1ms]
I170517 07:32:49.691567 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):55): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=55]
I170517 07:32:49.702377 45399 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):55: [(n1,s1):1 (n3,s3):3 (n2,s2):55]
I170517 07:32:49.711611 43444 storage/store.go:3152  [s2,r1/54:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:49.720175 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):55): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):55, next=56]
I170517 07:32:49.730381 45401 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):55: [(n1,s1):1 (n3,s3):3]
I170517 07:32:49.740348 43444 storage/store.go:3152  [s2,r1/55:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:49.740601 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 6774b7d2 at index 259
I170517 07:32:49.750795 43274 storage/store.go:2139  [replicaGC,s2,r1/55:/M{in-ax}] removing replica
I170517 07:32:49.755077 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 6943cc05 at index 259
I170517 07:32:49.757277 43274 storage/replica.go:684  [replicaGC,s2,r1/55:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:49.758472 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 09b1cff0 at index 259
W170517 07:32:49.761749 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):55: raft group deleted
I170517 07:32:49.774501 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 378, log entries: 64, rate-limit: 2.0 MiB/sec, 7ms
I170517 07:32:49.778319 45343 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 259 (id=09b1cff0, encoded size=67947, 1 rocksdb batches, 64 log entries)
I170517 07:32:49.788891 45343 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 10ms [clear=2ms batch=0ms entries=4ms commit=4ms]
I170517 07:32:49.791699 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):56): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=56]
I170517 07:32:49.801385 45437 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):56: [(n1,s1):1 (n3,s3):3 (n2,s2):56]
I170517 07:32:49.816310 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):56): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):56, next=57]
I170517 07:32:49.825471 45448 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):56: [(n1,s1):1 (n3,s3):3]
I170517 07:32:49.840941 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 84864c63 at index 263
I170517 07:32:49.846773 43444 storage/store.go:3152  [s2,r1/56:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:49.876801 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 384, log entries: 68, rate-limit: 2.0 MiB/sec, 34ms
I170517 07:32:49.878308 43274 storage/store.go:2139  [replicaGC,s2,r1/56:/M{in-ax}] removing replica
I170517 07:32:49.880085 43274 storage/replica.go:684  [replicaGC,s2,r1/56:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:49.881495 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):57): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=57]
I170517 07:32:49.888710 45441 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):57: [(n1,s1):1 (n3,s3):3 (n2,s2):57]
I170517 07:32:49.900838 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):57): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):57, next=58]
E170517 07:32:49.925966 43444 storage/store.go:3150  [s2,r1/57:{-}] unable to add to replica GC queue: replica not initialized
I170517 07:32:49.933614 45392 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):57: [(n1,s1):1 (n3,s3):3]
I170517 07:32:49.935085 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 58425982 at index 267
I170517 07:32:49.944176 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot e8e40331 at index 268
I170517 07:32:49.966380 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):57: kv pairs: 391, log entries: 72, rate-limit: 8.0 MiB/sec, 25ms
I170517 07:32:49.966974 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 44578353 at index 268
I170517 07:32:49.970651 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot f4eda0c4 at index 269
I170517 07:32:49.974953 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 38df160d at index 269
I170517 07:32:49.977369 45377 storage/replica_raftstorage.go:597  [s2,r1/57:{-}] applying Raft snapshot at index 267 (id=58425982, encoded size=74197, 1 rocksdb batches, 72 log entries)
I170517 07:32:49.980979 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 2e4e0240 at index 269
I170517 07:32:49.985312 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 888fa191 at index 269
I170517 07:32:49.991664 45377 storage/replica_raftstorage.go:605  [s2,r1/57:/M{in-ax}] applied Raft snapshot in 14ms [clear=1ms batch=0ms entries=11ms commit=1ms]
I170517 07:32:49.992871 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot e558ff9f at index 269
I170517 07:32:49.999458 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot f6d11e18 at index 269
I170517 07:32:50.001309 43274 storage/store.go:2139  [replicaGC,s2,r1/57:/M{in-ax}] removing replica
I170517 07:32:50.001844 43444 storage/store.go:3152  [s2,r1/57:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:50.003252 43274 storage/replica.go:684  [replicaGC,s2,r1/57:/M{in-ax}] removed 34 (25+9) keys in 2ms [clear=1ms commit=0ms]
I170517 07:32:50.004081 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 2ce8d6e3 at index 269
I170517 07:32:50.018369 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 392, log entries: 74, rate-limit: 2.0 MiB/sec, 14ms
I170517 07:32:50.032213 45473 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 269 (id=2ce8d6e3, encoded size=75307, 1 rocksdb batches, 74 log entries)
I170517 07:32:50.050802 45473 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 18ms [clear=1ms batch=0ms entries=14ms commit=2ms]
I170517 07:32:50.053758 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):58): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=58]
I170517 07:32:50.065560 45354 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):58: [(n1,s1):1 (n3,s3):3 (n2,s2):58]
I170517 07:32:50.079664 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):58): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):58, next=59]
I170517 07:32:50.094152 45531 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):58: [(n1,s1):1 (n3,s3):3]
I170517 07:32:50.102257 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 43e3f4ad at index 274
I170517 07:32:50.118807 43444 storage/store.go:3152  [s2,r1/58:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:50.120852 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 399, log entries: 79, rate-limit: 2.0 MiB/sec, 14ms
I170517 07:32:50.122064 43274 storage/store.go:2139  [replicaGC,s2,r1/58:/M{in-ax}] removing replica
I170517 07:32:50.123979 43274 storage/replica.go:684  [replicaGC,s2,r1/58:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:50.124947 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):59): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=59]
I170517 07:32:50.151894 45360 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):59: [(n1,s1):1 (n3,s3):3 (n2,s2):59]
I170517 07:32:50.167406 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):59): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):59, next=60]
I170517 07:32:50.176970 45602 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):59: [(n1,s1):1 (n3,s3):3]
E170517 07:32:50.187319 43444 storage/store.go:3150  [s2,r1/59:{-}] unable to add to replica GC queue: replica not initialized
I170517 07:32:50.187535 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 88f41665 at index 278
I170517 07:32:50.195284 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 405, log entries: 83, rate-limit: 2.0 MiB/sec, 7ms
I170517 07:32:50.198425 45547 storage/replica_raftstorage.go:597  [s2,r1/59:{-}] applying preemptive snapshot at index 278 (id=88f41665, encoded size=82181, 1 rocksdb batches, 83 log entries)
I170517 07:32:50.216546 45547 storage/replica_raftstorage.go:605  [s2,r1/59:/M{in-ax}] applied preemptive snapshot in 18ms [clear=2ms batch=0ms entries=13ms commit=1ms]
I170517 07:32:50.219494 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):60): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=60]
I170517 07:32:50.242223 45604 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):60: [(n1,s1):1 (n3,s3):3 (n2,s2):60]
I170517 07:32:50.257749 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):60): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):60, next=61]
I170517 07:32:50.274197 45635 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):60: [(n1,s1):1 (n3,s3):3]
I170517 07:32:50.286832 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 93b8fc94 at index 282
I170517 07:32:50.294788 43444 storage/store.go:3152  [s2,r1/59:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:50.296684 43274 storage/store.go:2139  [replicaGC,s2,r1/59:/M{in-ax}] removing replica
I170517 07:32:50.298467 43274 storage/replica.go:684  [replicaGC,s2,r1/59:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:50.308062 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 411, log entries: 87, rate-limit: 2.0 MiB/sec, 19ms
I170517 07:32:50.311988 45551 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 282 (id=93b8fc94, encoded size=85373, 1 rocksdb batches, 87 log entries)
I170517 07:32:50.324602 45551 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 12ms [clear=1ms batch=0ms entries=9ms commit=2ms]
I170517 07:32:50.329909 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):61): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=61]
I170517 07:32:50.370888 45573 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):61: [(n1,s1):1 (n3,s3):3 (n2,s2):61]
I170517 07:32:50.387745 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):61): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):61, next=62]
I170517 07:32:50.406309 45581 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):61: [(n1,s1):1 (n3,s3):3]
I170517 07:32:50.414937 43444 storage/store.go:3152  [s2,r1/61:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:50.424266 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot e61c59af at index 286
I170517 07:32:50.445646 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 417, log entries: 91, rate-limit: 2.0 MiB/sec, 20ms
I170517 07:32:50.447410 43274 storage/store.go:2139  [replicaGC,s2,r1/61:/M{in-ax}] removing replica
I170517 07:32:50.450700 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):62): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=62]
I170517 07:32:50.451657 43274 storage/replica.go:684  [replicaGC,s2,r1/61:/M{in-ax}] removed 33 (25+8) keys in 3ms [clear=3ms commit=0ms]
I170517 07:32:50.461885 45608 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):62: [(n1,s1):1 (n3,s3):3 (n2,s2):62]
I170517 07:32:50.474626 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):62): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):62, next=63]
I170517 07:32:50.490842 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 4fa9cee9 at index 289
I170517 07:32:50.496053 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):62: kv pairs: 420, log entries: 4, rate-limit: 8.0 MiB/sec, 5ms
I170517 07:32:50.498223 45611 storage/replica_raftstorage.go:597  [s2,r1/62:{-}] applying Raft snapshot at index 289 (id=4fa9cee9, encoded size=29966, 1 rocksdb batches, 4 log entries)
I170517 07:32:50.501527 45611 storage/replica_raftstorage.go:605  [s2,r1/62:/M{in-ax}] applied Raft snapshot in 3ms [clear=1ms batch=0ms entries=0ms commit=1ms]
I170517 07:32:50.503166 45670 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):62: [(n1,s1):1 (n3,s3):3]
I170517 07:32:50.511661 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 8fbc3efb at index 291
I170517 07:32:50.518609 43444 storage/store.go:3152  [s2,r1/62:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:50.521806 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 423, log entries: 6, rate-limit: 2.0 MiB/sec, 10ms
I170517 07:32:50.522962 43274 storage/store.go:2139  [replicaGC,s2,r1/62:/M{in-ax}] removing replica
I170517 07:32:50.524961 43274 storage/replica.go:684  [replicaGC,s2,r1/62:/M{in-ax}] removed 33 (25+8) keys in 2ms [clear=1ms commit=0ms]
I170517 07:32:50.527337 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):63): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=63]
I170517 07:32:50.535113 45699 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):63: [(n1,s1):1 (n3,s3):3 (n2,s2):63]
I170517 07:32:50.551014 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):63): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):63, next=64]
I170517 07:32:50.563369 45627 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):63: [(n1,s1):1 (n3,s3):3]
I170517 07:32:50.564891 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot c7de6bf0 at index 294
I170517 07:32:50.571283 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 819eb8c0 at index 295
I170517 07:32:50.579470 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 429, log entries: 10, rate-limit: 2.0 MiB/sec, 7ms
I170517 07:32:50.583236 45691 storage/replica_raftstorage.go:597  [s2,r1/63:{-}] applying preemptive snapshot at index 295 (id=819eb8c0, encoded size=34722, 1 rocksdb batches, 10 log entries)
I170517 07:32:50.586695 45691 storage/replica_raftstorage.go:605  [s2,r1/63:/M{in-ax}] applied preemptive snapshot in 3ms [clear=1ms batch=0ms entries=1ms commit=1ms]
I170517 07:32:50.591103 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):63: kv pairs: 429, log entries: 9, rate-limit: 8.0 MiB/sec, 25ms
I170517 07:32:50.594182 43444 storage/store.go:3152  [s2,r1/63:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:50.596041 43274 storage/store.go:2139  [replicaGC,s2,r1/63:/M{in-ax}] removing replica
I170517 07:32:50.597889 43274 storage/replica.go:684  [replicaGC,s2,r1/63:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:50.607611 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):64): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=64]
I170517 07:32:50.615916 45709 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):64: [(n1,s1):1 (n3,s3):3 (n2,s2):64]
I170517 07:32:50.621930 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):64): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):64, next=65]
I170517 07:32:50.631974 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot f64919a7 at index 297
I170517 07:32:50.643638 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):64: kv pairs: 432, log entries: 12, rate-limit: 8.0 MiB/sec, 10ms
I170517 07:32:50.649420 45694 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):64: [(n1,s1):1 (n3,s3):3]
I170517 07:32:50.657589 45599 storage/replica_raftstorage.go:597  [s2,r1/64:{-}] applying Raft snapshot at index 297 (id=f64919a7, encoded size=36350, 1 rocksdb batches, 12 log entries)
I170517 07:32:50.662148 45599 storage/replica_raftstorage.go:605  [s2,r1/64:/M{in-ax}] applied Raft snapshot in 4ms [clear=1ms batch=0ms entries=1ms commit=1ms]
I170517 07:32:50.664444 43444 storage/store.go:3152  [s2,r1/64:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:50.665963 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot fb527f0f at index 300
I170517 07:32:50.668217 43274 storage/store.go:2139  [replicaGC,s2,r1/64:/M{in-ax}] removing replica
I170517 07:32:50.670045 43274 storage/replica.go:684  [replicaGC,s2,r1/64:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=1ms commit=0ms]
I170517 07:32:50.670958 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 662e59c1 at index 300
I170517 07:32:50.677659 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 436, log entries: 15, rate-limit: 2.0 MiB/sec, 6ms
I170517 07:32:50.681323 45738 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 300 (id=662e59c1, encoded size=38404, 1 rocksdb batches, 15 log entries)
I170517 07:32:50.685468 45738 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 4ms [clear=1ms batch=0ms entries=1ms commit=1ms]
I170517 07:32:50.690724 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):65): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=65]
I170517 07:32:50.699914 45764 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):65: [(n1,s1):1 (n3,s3):3 (n2,s2):65]
I170517 07:32:50.712521 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):65): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):65, next=66]
I170517 07:32:50.725207 45780 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):65: [(n1,s1):1 (n3,s3):3]
I170517 07:32:50.749558 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 172254b2 at index 305
I170517 07:32:50.751686 43274 storage/store.go:2139  [replicaGC,s2,r1/65:/M{in-ax}] removing replica
I170517 07:32:50.753118 43444 storage/store.go:3152  [s2,r1/65:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:50.753641 43274 storage/replica.go:684  [replicaGC,s2,r1/65:/M{in-ax}] removed 33 (25+8) keys in 2ms [clear=1ms commit=0ms]
W170517 07:32:50.766774 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):65: raft group deleted
I170517 07:32:50.769264 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 443, log entries: 20, rate-limit: 2.0 MiB/sec, 10ms
I170517 07:32:50.772564 45782 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 305 (id=172254b2, encoded size=42086, 1 rocksdb batches, 20 log entries)
I170517 07:32:50.776832 45782 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 4ms [clear=1ms batch=0ms entries=1ms commit=1ms]
I170517 07:32:50.779520 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):66): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=66]
I170517 07:32:50.786930 45786 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):66: [(n1,s1):1 (n3,s3):3 (n2,s2):66]
I170517 07:32:50.828247 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):66): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):66, next=67]
I170517 07:32:50.846829 45827 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):66: [(n1,s1):1 (n3,s3):3]
I170517 07:32:50.855509 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 9cc3fe55 at index 310
I170517 07:32:50.858882 43444 storage/store.go:3152  [s2,r1/66:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:50.869732 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 450, log entries: 25, rate-limit: 2.0 MiB/sec, 11ms
I170517 07:32:50.874417 43274 storage/store.go:2139  [replicaGC,s2,r1/66:/M{in-ax}] removing replica
I170517 07:32:50.876804 43274 storage/replica.go:684  [replicaGC,s2,r1/66:/M{in-ax}] removed 33 (25+8) keys in 2ms [clear=1ms commit=0ms]
I170517 07:32:50.877031 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):67): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=67]
I170517 07:32:50.890797 45834 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):67: [(n1,s1):1 (n3,s3):3 (n2,s2):67]
I170517 07:32:50.913069 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):67): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):67, next=68]
I170517 07:32:50.924686 45836 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):67: [(n1,s1):1 (n3,s3):3]
I170517 07:32:50.932244 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot e4061d2e at index 313
I170517 07:32:50.937643 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 90cd79c3 at index 314
I170517 07:32:50.950206 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot ef61f831 at index 314
I170517 07:32:50.955228 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):67: kv pairs: 456, log entries: 28, rate-limit: 8.0 MiB/sec, 22ms
I170517 07:32:50.957106 45823 storage/replica_raftstorage.go:597  [s2,r1/67:{-}] applying Raft snapshot at index 313 (id=e4061d2e, encoded size=48340, 1 rocksdb batches, 28 log entries)
I170517 07:32:50.959441 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 0af1f701 at index 314
I170517 07:32:50.962332 45823 storage/replica_raftstorage.go:605  [s2,r1/67:/M{in-ax}] applied Raft snapshot in 5ms [clear=1ms batch=0ms entries=2ms commit=1ms]
I170517 07:32:50.965941 43444 storage/store.go:3152  [s2,r1/67:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:50.968013 43274 storage/store.go:2139  [replicaGC,s2,r1/67:/M{in-ax}] removing replica
I170517 07:32:50.969848 43274 storage/replica.go:684  [replicaGC,s2,r1/67:/M{in-ax}] removed 34 (25+9) keys in 1ms [clear=1ms commit=0ms]
W170517 07:32:50.972730 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):67: raft group deleted
I170517 07:32:50.973291 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 98f72f08 at index 314
I170517 07:32:50.993576 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 456, log entries: 29, rate-limit: 2.0 MiB/sec, 20ms
I170517 07:32:50.997882 45874 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 314 (id=98f72f08, encoded size=48960, 1 rocksdb batches, 29 log entries)
I170517 07:32:51.003498 45874 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 5ms [clear=1ms batch=0ms entries=2ms commit=1ms]
I170517 07:32:51.007919 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):68): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=68]
I170517 07:32:51.025481 45753 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):68: [(n1,s1):1 (n3,s3):3 (n2,s2):68]
I170517 07:32:51.053633 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):68): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):68, next=69]
I170517 07:32:51.063735 45867 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):68: [(n1,s1):1 (n3,s3):3]
I170517 07:32:51.076550 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot e87d1381 at index 318
I170517 07:32:51.089457 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 462, log entries: 33, rate-limit: 2.0 MiB/sec, 12ms
I170517 07:32:51.106352 43444 storage/store.go:3152  [s2,r1/68:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:51.108933 43274 storage/store.go:2139  [replicaGC,s2,r1/68:/M{in-ax}] removing replica
I170517 07:32:51.113692 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):69): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=69]
I170517 07:32:51.117898 43274 storage/replica.go:684  [replicaGC,s2,r1/68:/M{in-ax}] removed 33 (25+8) keys in 9ms [clear=8ms commit=0ms]
I170517 07:32:51.122194 45802 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):69: [(n1,s1):1 (n3,s3):3 (n2,s2):69]
I170517 07:32:51.132198 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):69): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):69, next=70]
I170517 07:32:51.157915 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot aea9bb64 at index 320
I170517 07:32:51.174991 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):69: kv pairs: 465, log entries: 35, rate-limit: 8.0 MiB/sec, 9ms
I170517 07:32:51.178354 45920 storage/replica_raftstorage.go:597  [s2,r1/69:{-}] applying Raft snapshot at index 320 (id=aea9bb64, encoded size=53780, 1 rocksdb batches, 35 log entries)
I170517 07:32:51.184397 45920 storage/replica_raftstorage.go:605  [s2,r1/69:/M{in-ax}] applied Raft snapshot in 6ms [clear=1ms batch=0ms entries=3ms commit=1ms]
I170517 07:32:51.190766 45755 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):69: [(n1,s1):1 (n3,s3):3]
I170517 07:32:51.195417 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot b932199f at index 322
I170517 07:32:51.214278 43444 storage/store.go:3152  [s2,r1/69:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:51.215323 43274 storage/store.go:2139  [replicaGC,s2,r1/69:/M{in-ax}] removing replica
I170517 07:32:51.217249 43274 storage/replica.go:684  [replicaGC,s2,r1/69:/M{in-ax}] removed 33 (25+8) keys in 2ms [clear=1ms commit=0ms]
I170517 07:32:51.224565 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 468, log entries: 37, rate-limit: 2.0 MiB/sec, 7ms
I170517 07:32:51.228253 45893 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 322 (id=b932199f, encoded size=55344, 1 rocksdb batches, 37 log entries)
I170517 07:32:51.254656 45893 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 26ms [clear=1ms batch=2ms entries=5ms commit=15ms]
I170517 07:32:51.257730 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):70): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=70]
I170517 07:32:51.265390 45872 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):70: [(n1,s1):1 (n3,s3):3 (n2,s2):70]
I170517 07:32:51.272693 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):70): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):70, next=71]
I170517 07:32:51.282556 45941 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):70: [(n1,s1):1 (n3,s3):3]
I170517 07:32:51.296406 43444 storage/store.go:3152  [s2,r1/70:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:51.299805 43274 storage/store.go:2139  [replicaGC,s2,r1/70:/M{in-ax}] removing replica
I170517 07:32:51.302093 43274 storage/replica.go:684  [replicaGC,s2,r1/70:/M{in-ax}] removed 34 (25+9) keys in 2ms [clear=1ms commit=0ms]
I170517 07:32:51.303229 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 050b8804 at index 326
I170517 07:32:51.316994 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 474, log entries: 41, rate-limit: 2.0 MiB/sec, 12ms
I170517 07:32:51.322498 45851 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 326 (id=050b8804, encoded size=58536, 1 rocksdb batches, 41 log entries)
I170517 07:32:51.329928 45851 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 7ms [clear=1ms batch=0ms entries=4ms commit=1ms]
I170517 07:32:51.332831 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):71): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=71]
I170517 07:32:51.340844 45944 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):71: [(n1,s1):1 (n3,s3):3 (n2,s2):71]
I170517 07:32:51.353535 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):71): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):71, next=72]
I170517 07:32:51.378542 45972 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):71: [(n1,s1):1 (n3,s3):3]
I170517 07:32:51.395085 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 92087f84 at index 331
I170517 07:32:51.409705 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 481, log entries: 46, rate-limit: 2.0 MiB/sec, 14ms
I170517 07:32:51.418071 43444 storage/store.go:3152  [s2,r1/71:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:51.421799 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):72): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=72]
I170517 07:32:51.422219 43274 storage/store.go:2139  [replicaGC,s2,r1/71:/M{in-ax}] removing replica
I170517 07:32:51.445772 43274 storage/replica.go:684  [replicaGC,s2,r1/71:/M{in-ax}] removed 33 (25+8) keys in 18ms [clear=10ms commit=8ms]
I170517 07:32:51.447454 45990 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):72: [(n1,s1):1 (n3,s3):3 (n2,s2):72]
I170517 07:32:51.466754 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):72): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):72, next=73]
I170517 07:32:51.472235 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 9273ad4d at index 334
I170517 07:32:51.481532 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):72: kv pairs: 485, log entries: 49, rate-limit: 8.0 MiB/sec, 9ms
I170517 07:32:51.485051 46000 storage/replica_raftstorage.go:597  [s2,r1/72:{-}] applying Raft snapshot at index 334 (id=9273ad4d, encoded size=64336, 1 rocksdb batches, 49 log entries)
I170517 07:32:51.498394 46050 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):72: [(n1,s1):1 (n3,s3):3]
I170517 07:32:51.519357 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 4faac6d9 at index 337
I170517 07:32:51.520805 46000 storage/replica_raftstorage.go:605  [s2,r1/72:/M{in-ax}] applied Raft snapshot in 36ms [clear=1ms batch=0ms entries=16ms commit=16ms]
I170517 07:32:51.523168 43444 storage/store.go:3152  [s2,r1/72:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:51.553411 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 489, log entries: 52, rate-limit: 2.0 MiB/sec, 31ms
I170517 07:32:51.555546 46036 storage/replica_raftstorage.go:597  [s2,r1/72:/M{in-ax}] applying preemptive snapshot at index 337 (id=4faac6d9, encoded size=66390, 1 rocksdb batches, 52 log entries)
I170517 07:32:51.564822 46036 storage/replica_raftstorage.go:605  [s2,r1/72:/M{in-ax}] applied preemptive snapshot in 9ms [clear=1ms batch=0ms entries=5ms commit=2ms]
I170517 07:32:51.566905 43274 storage/store.go:2139  [replicaGC,s2,r1/72:/M{in-ax}] removing replica
I170517 07:32:51.568100 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):73): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=73]
I170517 07:32:51.570757 43274 storage/replica.go:684  [replicaGC,s2,r1/72:/M{in-ax}] removed 33 (25+8) keys in 3ms [clear=3ms commit=0ms]
I170517 07:32:51.577124 46067 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):73: [(n1,s1):1 (n3,s3):3 (n2,s2):73]
I170517 07:32:51.588156 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):73): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):73, next=74]
I170517 07:32:51.599873 46032 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):73: [(n1,s1):1 (n3,s3):3]
I170517 07:32:51.603384 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 7257f983 at index 340
E170517 07:32:51.613529 43444 storage/store.go:3150  [s2,r1/73:{-}] unable to add to replica GC queue: replica not initialized
I170517 07:32:51.621290 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot f4d63803 at index 341
I170517 07:32:51.623804 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 7f4ed8aa at index 341
I170517 07:32:51.624211 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):73: kv pairs: 495, log entries: 55, rate-limit: 8.0 MiB/sec, 20ms
I170517 07:32:51.632145 45981 storage/replica_raftstorage.go:597  [s2,r1/73:{-}] applying Raft snapshot at index 340 (id=7257f983, encoded size=68962, 1 rocksdb batches, 55 log entries)
I170517 07:32:51.635971 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 670ecedc at index 341
I170517 07:32:51.644765 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 73aa0133 at index 341
I170517 07:32:51.647918 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 438fefd9 at index 341
I170517 07:32:51.650811 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot a73f5469 at index 341
I170517 07:32:51.654850 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 9cd5914e at index 341
I170517 07:32:51.655932 45981 storage/replica_raftstorage.go:605  [s2,r1/73:/M{in-ax}] applied Raft snapshot in 24ms [clear=1ms batch=0ms entries=18ms commit=1ms]
I170517 07:32:51.657821 43444 storage/store.go:3152  [s2,r1/73:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:51.660259 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 26629139 at index 341
I170517 07:32:51.662657 43274 storage/store.go:2139  [replicaGC,s2,r1/73:/M{in-ax}] removing replica
I170517 07:32:51.663402 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot fc0e3932 at index 341
I170517 07:32:51.665824 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot ea2079f5 at index 341
I170517 07:32:51.668282 43274 storage/replica.go:684  [replicaGC,s2,r1/73:/M{in-ax}] removed 34 (25+9) keys in 5ms [clear=4ms commit=0ms]
I170517 07:32:51.668523 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot af2d8bd2 at index 341
I170517 07:32:51.679027 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 495, log entries: 56, rate-limit: 2.0 MiB/sec, 10ms
I170517 07:32:51.682959 45985 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 341 (id=af2d8bd2, encoded size=69582, 1 rocksdb batches, 56 log entries)
I170517 07:32:51.694171 45985 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 11ms [clear=1ms batch=0ms entries=8ms commit=1ms]
I170517 07:32:51.698648 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):74): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=74]
I170517 07:32:51.712668 46115 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):74: [(n1,s1):1 (n3,s3):3 (n2,s2):74]
I170517 07:32:51.728652 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):74): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):74, next=75]
I170517 07:32:51.743053 46104 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):74: [(n1,s1):1 (n3,s3):3]
I170517 07:32:51.771927 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 3e0dda40 at index 346
I170517 07:32:51.772786 43274 storage/store.go:2139  [replicaGC,s2,r1/74:/M{in-ax}] removing replica
I170517 07:32:51.777575 43444 storage/store.go:3152  [s2,r1/74:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:51.779373 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 1e78131c at index 346
I170517 07:32:51.790910 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 7eef77ac at index 346
I170517 07:32:51.792539 43274 storage/replica.go:684  [replicaGC,s2,r1/74:/M{in-ax}] removed 33 (25+8) keys in 19ms [clear=6ms commit=13ms]
I170517 07:32:51.793627 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot ebf59e0f at index 346
I170517 07:32:51.804832 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 501, log entries: 61, rate-limit: 2.0 MiB/sec, 11ms
I170517 07:32:51.809643 46078 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 346 (id=ebf59e0f, encoded size=72792, 1 rocksdb batches, 61 log entries)
I170517 07:32:51.826618 46078 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 17ms [clear=2ms batch=0ms entries=4ms commit=10ms]
I170517 07:32:51.833495 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):75): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=75]
I170517 07:32:51.847537 46139 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):75: [(n1,s1):1 (n3,s3):3 (n2,s2):75]
I170517 07:32:51.855666 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):75): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):75, next=76]
I170517 07:32:51.863524 46011 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):75: [(n1,s1):1 (n3,s3):3]
I170517 07:32:51.874563 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 89fb84b3 at index 350
I170517 07:32:51.876355 43274 storage/store.go:2139  [replicaGC,s2,r1/75:/M{in-ax}] removing replica
I170517 07:32:51.879885 43274 storage/replica.go:684  [replicaGC,s2,r1/75:/M{in-ax}] removed 33 (25+8) keys in 3ms [clear=2ms commit=1ms]
I170517 07:32:51.910162 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 507, log entries: 65, rate-limit: 2.0 MiB/sec, 35ms
I170517 07:32:51.920633 46064 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 350 (id=89fb84b3, encoded size=75984, 1 rocksdb batches, 65 log entries)
I170517 07:32:51.955283 46064 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 34ms [clear=2ms batch=0ms entries=20ms commit=12ms]
I170517 07:32:51.959015 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):76): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=76]
I170517 07:32:51.966815 46091 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):76: [(n1,s1):1 (n3,s3):3 (n2,s2):76]
I170517 07:32:51.973648 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):76): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):76, next=77]
I170517 07:32:52.009955 46126 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):76: [(n1,s1):1 (n3,s3):3]
I170517 07:32:52.030415 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 1b2d0d71 at index 354
I170517 07:32:52.034962 43444 storage/store.go:3152  [s2,r1/76:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:52.047186 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 513, log entries: 69, rate-limit: 2.0 MiB/sec, 16ms
I170517 07:32:52.048698 43274 storage/store.go:2139  [replicaGC,s2,r1/76:/M{in-ax}] removing replica
I170517 07:32:52.051461 43274 storage/replica.go:684  [replicaGC,s2,r1/76:/M{in-ax}] removed 33 (25+8) keys in 2ms [clear=1ms commit=1ms]
I170517 07:32:52.052302 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):77): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=77]
W170517 07:32:52.061205 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):76: raft group deleted
I170517 07:32:52.068659 46167 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):77: [(n1,s1):1 (n3,s3):3 (n2,s2):77]
I170517 07:32:52.085811 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):77): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):77, next=78]
I170517 07:32:52.108782 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 9206dec7 at index 357
I170517 07:32:52.140810 46229 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):77: [(n1,s1):1 (n3,s3):3]
I170517 07:32:52.148977 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):77: kv pairs: 517, log entries: 72, rate-limit: 8.0 MiB/sec, 40ms
I170517 07:32:52.156290 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 5357cd5b at index 359
I170517 07:32:52.159523 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 4ae22b88 at index 359
I170517 07:32:52.161028 46227 storage/replica_raftstorage.go:597  [s2,r1/77:{-}] applying Raft snapshot at index 357 (id=9206dec7, encoded size=81294, 1 rocksdb batches, 72 log entries)
I170517 07:32:52.165906 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 631fdefa at index 359
I170517 07:32:52.168751 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 561e0883 at index 359
I170517 07:32:52.171748 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 97372c80 at index 359
I170517 07:32:52.173199 46227 storage/replica_raftstorage.go:605  [s2,r1/77:/M{in-ax}] applied Raft snapshot in 12ms [clear=2ms batch=0ms entries=8ms commit=1ms]
I170517 07:32:52.176908 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 2efa34f4 at index 359
I170517 07:32:52.177171 43444 storage/store.go:3152  [s2,r1/77:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:52.195986 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 520, log entries: 74, rate-limit: 2.0 MiB/sec, 18ms
I170517 07:32:52.198310 46203 storage/replica_raftstorage.go:597  [s2,r1/77:/M{in-ax}] applying preemptive snapshot at index 359 (id=2efa34f4, encoded size=82858, 1 rocksdb batches, 74 log entries)
I170517 07:32:52.222334 43444 storage/store.go:3152  [s2,r1/77:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:52.232241 46203 storage/replica_raftstorage.go:605  [s2,r1/77:/M{in-ax}] applied preemptive snapshot in 34ms [clear=7ms batch=0ms entries=6ms commit=20ms]
I170517 07:32:52.233299 43274 storage/store.go:2139  [replicaGC,s2,r1/77:/M{in-ax}] removing replica
I170517 07:32:52.243180 43274 storage/replica.go:684  [replicaGC,s2,r1/77:/M{in-ax}] removed 33 (25+8) keys in 4ms [clear=3ms commit=1ms]
I170517 07:32:52.245385 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):78): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=78]
I170517 07:32:52.254302 46290 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):78: [(n1,s1):1 (n3,s3):3 (n2,s2):78]
I170517 07:32:52.282714 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):78): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):78, next=79]
I170517 07:32:52.296258 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 1804068f at index 363
I170517 07:32:52.315769 46277 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):78: [(n1,s1):1 (n3,s3):3]
I170517 07:32:52.318154 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):78: kv pairs: 525, log entries: 78, rate-limit: 8.0 MiB/sec, 21ms
I170517 07:32:52.320750 46293 storage/replica_raftstorage.go:597  [s2,r1/78:{-}] applying Raft snapshot at index 363 (id=1804068f, encoded size=85466, 1 rocksdb batches, 78 log entries)
I170517 07:32:52.325599 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 7fda48df at index 365
I170517 07:32:52.331654 46293 storage/replica_raftstorage.go:605  [s2,r1/78:/M{in-ax}] applied Raft snapshot in 11ms [clear=1ms batch=0ms entries=7ms commit=2ms]
I170517 07:32:52.332261 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot ef19e625 at index 365
I170517 07:32:52.341459 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 528, log entries: 80, rate-limit: 2.0 MiB/sec, 9ms
I170517 07:32:52.344073 43444 storage/store.go:3152  [s2,r1/78:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:52.361562 46223 storage/replica_raftstorage.go:597  [s2,r1/78:/M{in-ax}] applying preemptive snapshot at index 365 (id=ef19e625, encoded size=87030, 1 rocksdb batches, 80 log entries)
I170517 07:32:52.371531 46223 storage/replica_raftstorage.go:605  [s2,r1/78:/M{in-ax}] applied preemptive snapshot in 10ms [clear=2ms batch=0ms entries=5ms commit=2ms]
I170517 07:32:52.372857 43274 storage/store.go:2139  [replicaGC,s2,r1/78:/M{in-ax}] removing replica
I170517 07:32:52.375124 43274 storage/replica.go:684  [replicaGC,s2,r1/78:/M{in-ax}] removed 33 (25+8) keys in 2ms [clear=1ms commit=1ms]
I170517 07:32:52.375487 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):79): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=79]
I170517 07:32:52.384679 46225 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):79: [(n1,s1):1 (n3,s3):3 (n2,s2):79]
I170517 07:32:52.396484 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):79): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):79, next=80]
I170517 07:32:52.409905 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 1805ec89 at index 368
I170517 07:32:52.411737 46298 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):79: [(n1,s1):1 (n3,s3):3]
E170517 07:32:52.412017 43444 storage/store.go:3150  [s2,r1/79:{-}] unable to add to replica GC queue: replica not initialized
I170517 07:32:52.417181 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):79: kv pairs: 534, log entries: 83, rate-limit: 8.0 MiB/sec, 6ms
I170517 07:32:52.424906 46343 storage/replica_raftstorage.go:597  [s2,r1/79:{-}] applying Raft snapshot at index 368 (id=1805ec89, encoded size=89602, 1 rocksdb batches, 83 log entries)
I170517 07:32:52.430937 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 3216861e at index 369
I170517 07:32:52.441872 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 628c0407 at index 369
I170517 07:32:52.446505 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 0bbd2b70 at index 369
I170517 07:32:52.450059 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot a4814b17 at index 369
I170517 07:32:52.454396 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot b58fc257 at index 369
I170517 07:32:52.462172 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot a2534adb at index 369
I170517 07:32:52.462615 46343 storage/replica_raftstorage.go:605  [s2,r1/79:/M{in-ax}] applied Raft snapshot in 38ms [clear=1ms batch=0ms entries=25ms commit=6ms]
I170517 07:32:52.464649 43444 storage/store.go:3152  [s2,r1/79:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:52.478093 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 534, log entries: 84, rate-limit: 2.0 MiB/sec, 15ms
I170517 07:32:52.490749 46286 storage/replica_raftstorage.go:597  [s2,r1/79:/M{in-ax}] applying preemptive snapshot at index 369 (id=a2534adb, encoded size=90222, 1 rocksdb batches, 84 log entries)
I170517 07:32:52.503772 46286 storage/replica_raftstorage.go:605  [s2,r1/79:/M{in-ax}] applied preemptive snapshot in 13ms [clear=2ms batch=0ms entries=8ms commit=2ms]
I170517 07:32:52.504204 43274 storage/store.go:2139  [replicaGC,s2,r1/79:/M{in-ax}] removing replica
I170517 07:32:52.507775 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):80): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=80]
I170517 07:32:52.514881 43274 storage/replica.go:684  [replicaGC,s2,r1/79:/M{in-ax}] removed 33 (25+8) keys in 10ms [clear=10ms commit=1ms]
I170517 07:32:52.529606 46370 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):80: [(n1,s1):1 (n3,s3):3 (n2,s2):80]
I170517 07:32:52.543726 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):80): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):80, next=81]
I170517 07:32:52.555485 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot f07c9160 at index 372
I170517 07:32:52.562016 46375 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):80: [(n1,s1):1 (n3,s3):3]
I170517 07:32:52.577938 46311 storage/replica_raftstorage.go:597  [s2,r1/80:{-}] applying Raft snapshot at index 372 (id=f07c9160, encoded size=92794, 1 rocksdb batches, 87 log entries)
I170517 07:32:52.579539 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 8bd64329 at index 373
I170517 07:32:52.590121 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):80: kv pairs: 540, log entries: 87, rate-limit: 8.0 MiB/sec, 34ms
I170517 07:32:52.593939 46311 storage/replica_raftstorage.go:605  [s2,r1/80:/M{in-ax}] applied Raft snapshot in 15ms [clear=2ms batch=0ms entries=8ms commit=3ms]
I170517 07:32:52.594617 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 8e16e89f at index 373
I170517 07:32:52.596189 43444 storage/store.go:3152  [s2,r1/80:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:52.598363 43274 storage/store.go:2139  [replicaGC,s2,r1/80:/M{in-ax}] removing replica
I170517 07:32:52.600638 43274 storage/replica.go:684  [replicaGC,s2,r1/80:/M{in-ax}] removed 34 (25+9) keys in 2ms [clear=1ms commit=0ms]
I170517 07:32:52.607887 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 540, log entries: 88, rate-limit: 2.0 MiB/sec, 10ms
I170517 07:32:52.612253 46383 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 373 (id=8e16e89f, encoded size=93414, 1 rocksdb batches, 88 log entries)
I170517 07:32:52.628186 46383 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 16ms [clear=1ms batch=0ms entries=12ms commit=2ms]
I170517 07:32:52.636376 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):81): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=81]
I170517 07:32:52.646808 46368 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):81: [(n1,s1):1 (n3,s3):3 (n2,s2):81]
I170517 07:32:52.656347 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):81): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):81, next=82]
I170517 07:32:52.671595 46176 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):81: [(n1,s1):1 (n3,s3):3]
I170517 07:32:52.682467 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 37e36e0d at index 377
I170517 07:32:52.701862 43444 storage/store.go:3152  [s2,r1/81:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:52.702232 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 546, log entries: 92, rate-limit: 2.0 MiB/sec, 15ms
I170517 07:32:52.703640 43274 storage/store.go:2139  [replicaGC,s2,r1/81:/M{in-ax}] removing replica
I170517 07:32:52.707981 43274 storage/replica.go:684  [replicaGC,s2,r1/81:/M{in-ax}] removed 33 (25+8) keys in 4ms [clear=1ms commit=2ms]
I170517 07:32:52.711807 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):82): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=82]
I170517 07:32:52.725560 46435 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):82: [(n1,s1):1 (n3,s3):3 (n2,s2):82]
I170517 07:32:52.736941 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):82): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):82, next=83]
I170517 07:32:52.751048 46422 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):82: [(n1,s1):1 (n3,s3):3]
I170517 07:32:52.753666 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot df404587 at index 381
I170517 07:32:52.762521 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 202f8c1c at index 382
I170517 07:32:52.769131 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot d5330ec2 at index 382
I170517 07:32:52.776691 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):82: kv pairs: 552, log entries: 6, rate-limit: 8.0 MiB/sec, 22ms
I170517 07:32:52.788041 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 343d40e9 at index 382
I170517 07:32:52.790289 46467 storage/replica_raftstorage.go:597  [s2,r1/82:{-}] applying Raft snapshot at index 381 (id=df404587, encoded size=38848, 1 rocksdb batches, 6 log entries)
I170517 07:32:52.794977 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 7b239d96 at index 383
I170517 07:32:52.798214 46467 storage/replica_raftstorage.go:605  [s2,r1/82:/M{in-ax}] applied Raft snapshot in 8ms [clear=2ms batch=0ms entries=1ms commit=2ms]
I170517 07:32:52.798378 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot cf9050cc at index 383
I170517 07:32:52.800235 43444 storage/store.go:3152  [s2,r1/82:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:52.811646 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 553, log entries: 8, rate-limit: 2.0 MiB/sec, 13ms
I170517 07:32:52.814435 46391 storage/replica_raftstorage.go:597  [s2,r1/82:/M{in-ax}] applying preemptive snapshot at index 383 (id=cf9050cc, encoded size=39958, 1 rocksdb batches, 8 log entries)
I170517 07:32:52.819459 46391 storage/replica_raftstorage.go:605  [s2,r1/82:/M{in-ax}] applied preemptive snapshot in 5ms [clear=1ms batch=0ms entries=1ms commit=1ms]
I170517 07:32:52.820111 43274 storage/store.go:2139  [replicaGC,s2,r1/82:/M{in-ax}] removing replica
I170517 07:32:52.822644 43274 storage/replica.go:684  [replicaGC,s2,r1/82:/M{in-ax}] removed 33 (25+8) keys in 2ms [clear=2ms commit=0ms]
I170517 07:32:52.823660 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):83): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=83]
I170517 07:32:52.839277 46412 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):83: [(n1,s1):1 (n3,s3):3 (n2,s2):83]
I170517 07:32:52.871473 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):83): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):83, next=84]
I170517 07:32:52.881223 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot edd47432 at index 387
I170517 07:32:52.884466 46498 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):83: [(n1,s1):1 (n3,s3):3]
I170517 07:32:52.894201 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 3d9b1fe2 at index 388
I170517 07:32:52.897483 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 0d87d499 at index 388
I170517 07:32:52.921385 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):83: kv pairs: 560, log entries: 12, rate-limit: 8.0 MiB/sec, 39ms
I170517 07:32:52.929544 46461 storage/replica_raftstorage.go:597  [s2,r1/83:{-}] applying Raft snapshot at index 387 (id=edd47432, encoded size=43020, 1 rocksdb batches, 12 log entries)
I170517 07:32:52.931439 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot e73bfe49 at index 388
I170517 07:32:52.934163 46461 storage/replica_raftstorage.go:605  [s2,r1/83:/M{in-ax}] applied Raft snapshot in 4ms [clear=1ms batch=0ms entries=1ms commit=1ms]
I170517 07:32:52.936497 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 5b4bb0f4 at index 388
I170517 07:32:52.946512 43444 storage/store.go:3152  [s2,r1/83:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:52.951814 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 560, log entries: 13, rate-limit: 2.0 MiB/sec, 13ms
I170517 07:32:52.957645 46442 storage/replica_raftstorage.go:597  [s2,r1/83:/M{in-ax}] applying preemptive snapshot at index 388 (id=5b4bb0f4, encoded size=43640, 1 rocksdb batches, 13 log entries)
I170517 07:32:52.963402 46442 storage/replica_raftstorage.go:605  [s2,r1/83:/M{in-ax}] applied preemptive snapshot in 6ms [clear=1ms batch=0ms entries=2ms commit=1ms]
I170517 07:32:52.964005 43274 storage/store.go:2139  [replicaGC,s2,r1/83:/M{in-ax}] removing replica
I170517 07:32:52.965949 43274 storage/replica.go:684  [replicaGC,s2,r1/83:/M{in-ax}] removed 33 (25+8) keys in 2ms [clear=1ms commit=0ms]
I170517 07:32:52.969119 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):84): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=84]
I170517 07:32:52.976776 46446 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):84: [(n1,s1):1 (n3,s3):3 (n2,s2):84]
I170517 07:32:53.003725 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):84): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):84, next=85]
I170517 07:32:53.020778 46510 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):84: [(n1,s1):1 (n3,s3):3]
I170517 07:32:53.030876 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 183b220b at index 392
I170517 07:32:53.040834 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 3c1e8c7f at index 393
I170517 07:32:53.044773 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot f8c80912 at index 393
I170517 07:32:53.045462 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):84: kv pairs: 567, log entries: 17, rate-limit: 8.0 MiB/sec, 14ms
I170517 07:32:53.053794 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 1f5487cc at index 393
I170517 07:32:53.058930 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 9af3247c at index 393
I170517 07:32:53.063892 46512 storage/replica_raftstorage.go:597  [s2,r1/84:{-}] applying Raft snapshot at index 392 (id=183b220b, encoded size=46702, 1 rocksdb batches, 17 log entries)
I170517 07:32:53.068768 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 72c0b145 at index 393
I170517 07:32:53.071873 46512 storage/replica_raftstorage.go:605  [s2,r1/84:/M{in-ax}] applied Raft snapshot in 8ms [clear=1ms batch=0ms entries=3ms commit=3ms]
I170517 07:32:53.073722 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 0e698b9c at index 393
I170517 07:32:53.074232 43444 storage/store.go:3152  [s2,r1/84:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:53.082947 43274 storage/store.go:2139  [replicaGC,s2,r1/84:/M{in-ax}] removing replica
I170517 07:32:53.085403 43274 storage/replica.go:684  [replicaGC,s2,r1/84:/M{in-ax}] removed 34 (25+9) keys in 2ms [clear=2ms commit=0ms]
I170517 07:32:53.095344 46550 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 393 (id=0e698b9c, encoded size=47322, 1 rocksdb batches, 18 log entries)
I170517 07:32:53.099712 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 567, log entries: 18, rate-limit: 2.0 MiB/sec, 18ms
I170517 07:32:53.100349 46550 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 5ms [clear=1ms batch=0ms entries=1ms commit=1ms]
I170517 07:32:53.104336 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):85): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=85]
I170517 07:32:53.111512 43444 storage/store.go:3152  [s2,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:53.113378 43274 storage/store.go:2139  [replicaGC,s2,r1/?:/M{in-ax}] removing replica
I170517 07:32:53.113536 46567 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):85: [(n1,s1):1 (n3,s3):3 (n2,s2):85]
I170517 07:32:53.126480 43274 storage/replica.go:684  [replicaGC,s2,r1/?:/M{in-ax}] removed 33 (25+8) keys in 13ms [clear=3ms commit=10ms]
I170517 07:32:53.128427 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):85): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):85, next=86]
I170517 07:32:53.144965 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 6384b1fa at index 396
I170517 07:32:53.146863 46541 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):85: [(n1,s1):1 (n3,s3):3]
I170517 07:32:53.152139 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):85: kv pairs: 573, log entries: 21, rate-limit: 8.0 MiB/sec, 7ms
I170517 07:32:53.158089 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 35654fc0 at index 397
I170517 07:32:53.160370 46594 storage/replica_raftstorage.go:597  [s2,r1/85:{-}] applying Raft snapshot at index 396 (id=6384b1fa, encoded size=49894, 1 rocksdb batches, 21 log entries)
I170517 07:32:53.161315 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 2653dfa8 at index 397
I170517 07:32:53.164520 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot e4781b89 at index 397
I170517 07:32:53.167209 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 4efba16c at index 397
I170517 07:32:53.170167 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 89dbee83 at index 397
I170517 07:32:53.172993 46594 storage/replica_raftstorage.go:605  [s2,r1/85:/M{in-ax}] applied Raft snapshot in 12ms [clear=4ms batch=0ms entries=6ms commit=2ms]
I170517 07:32:53.173096 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 3e8788b2 at index 397
I170517 07:32:53.175006 43444 storage/store.go:3152  [s2,r1/85:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:53.180806 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 573, log entries: 22, rate-limit: 2.0 MiB/sec, 7ms
I170517 07:32:53.181769 46581 storage/replica_raftstorage.go:597  [s2,r1/85:/M{in-ax}] applying preemptive snapshot at index 397 (id=3e8788b2, encoded size=50514, 1 rocksdb batches, 22 log entries)
I170517 07:32:53.187281 46581 storage/replica_raftstorage.go:605  [s2,r1/85:/M{in-ax}] applied preemptive snapshot in 5ms [clear=1ms batch=0ms entries=2ms commit=2ms]
I170517 07:32:53.187884 43274 storage/store.go:2139  [replicaGC,s2,r1/85:/M{in-ax}] removing replica
I170517 07:32:53.190536 43274 storage/replica.go:684  [replicaGC,s2,r1/85:/M{in-ax}] removed 33 (25+8) keys in 2ms [clear=2ms commit=0ms]
I170517 07:32:53.190688 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):86): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=86]
I170517 07:32:53.197805 46605 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):86: [(n1,s1):1 (n3,s3):3 (n2,s2):86]
I170517 07:32:53.213315 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):86): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):86, next=87]
I170517 07:32:53.265720 46609 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):86: [(n1,s1):1 (n3,s3):3]
I170517 07:32:53.285522 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 55420006 at index 401
E170517 07:32:53.291430 43444 storage/store.go:3150  [s2,r1/86:{-}] unable to add to replica GC queue: replica not initialized
I170517 07:32:53.329744 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 579, log entries: 26, rate-limit: 2.0 MiB/sec, 44ms
I170517 07:32:53.336884 46647 storage/replica_raftstorage.go:597  [s2,r1/86:{-}] applying preemptive snapshot at index 401 (id=55420006, encoded size=53702, 1 rocksdb batches, 26 log entries)
E170517 07:32:53.342004 43444 storage/store.go:3150  [s2,r1/86:{-}] unable to add to replica GC queue: replica not initialized
I170517 07:32:53.345156 46647 storage/replica_raftstorage.go:605  [s2,r1/86:/M{in-ax}] applied preemptive snapshot in 8ms [clear=1ms batch=0ms entries=3ms commit=3ms]
I170517 07:32:53.350285 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):87): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=87]
I170517 07:32:53.359827 46613 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):87: [(n1,s1):1 (n3,s3):3 (n2,s2):87]
I170517 07:32:53.374041 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):87): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):87, next=88]
I170517 07:32:53.385688 46675 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):87: [(n1,s1):1 (n3,s3):3]
I170517 07:32:53.395604 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 459b68db at index 405
I170517 07:32:53.401272 43444 storage/store.go:3152  [s2,r1/87:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:53.410539 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 585, log entries: 30, rate-limit: 2.0 MiB/sec, 8ms
I170517 07:32:53.412384 43274 storage/store.go:2139  [replicaGC,s2,r1/87:/M{in-ax}] removing replica
I170517 07:32:53.414768 43274 storage/replica.go:684  [replicaGC,s2,r1/87:/M{in-ax}] removed 33 (25+8) keys in 2ms [clear=1ms commit=0ms]
I170517 07:32:53.414919 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):88): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=88]
I170517 07:32:53.423008 46710 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):88: [(n1,s1):1 (n3,s3):3 (n2,s2):88]
I170517 07:32:53.437209 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):88): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):88, next=89]
I170517 07:32:53.446254 46715 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):88: [(n1,s1):1 (n3,s3):3]
I170517 07:32:53.456093 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 0c6a480f at index 409
I170517 07:32:53.468120 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 591, log entries: 34, rate-limit: 2.0 MiB/sec, 11ms
E170517 07:32:53.477780 43444 storage/store.go:3150  [s2,r1/88:{-}] unable to add to replica GC queue: replica not initialized
I170517 07:32:53.495834 46621 storage/replica_raftstorage.go:597  [s2,r1/88:{-}] applying preemptive snapshot at index 409 (id=0c6a480f, encoded size=60086, 1 rocksdb batches, 34 log entries)
I170517 07:32:53.521689 46621 storage/replica_raftstorage.go:605  [s2,r1/88:/M{in-ax}] applied preemptive snapshot in 24ms [clear=3ms batch=0ms entries=11ms commit=8ms]
I170517 07:32:53.524519 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):89): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=89]
I170517 07:32:53.534403 46698 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):89: [(n1,s1):1 (n3,s3):3 (n2,s2):89]
I170517 07:32:53.545926 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):89): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):89, next=90]
I170517 07:32:53.605160 46718 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):89: [(n1,s1):1 (n3,s3):3]
I170517 07:32:53.610797 43444 storage/store.go:3152  [s2,r1/89:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:53.612748 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot f448f85d at index 415
I170517 07:32:53.626854 43274 storage/store.go:2139  [replicaGC,s2,r1/89:/M{in-ax}] removing replica
I170517 07:32:53.629200 43274 storage/replica.go:684  [replicaGC,s2,r1/89:/M{in-ax}] removed 33 (25+8) keys in 2ms [clear=2ms commit=0ms]
I170517 07:32:53.629632 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 599, log entries: 40, rate-limit: 2.0 MiB/sec, 16ms
I170517 07:32:53.632173 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):90): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=90]
I170517 07:32:53.648388 46786 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):90: [(n1,s1):1 (n3,s3):3 (n2,s2):90]
I170517 07:32:53.657065 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):90): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):90, next=91]
I170517 07:32:53.673717 46772 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):90: [(n1,s1):1 (n3,s3):3]
E170517 07:32:53.679638 43444 storage/store.go:3150  [s2,r1/90:{-}] unable to add to replica GC queue: replica not initialized
I170517 07:32:53.680633 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c7e2db2a at index 420
I170517 07:32:53.688621 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 606, log entries: 45, rate-limit: 2.0 MiB/sec, 6ms
I170517 07:32:53.691584 46777 storage/replica_raftstorage.go:597  [s2,r1/90:{-}] applying preemptive snapshot at index 420 (id=c7e2db2a, encoded size=67940, 1 rocksdb batches, 45 log entries)
I170517 07:32:53.698239 46777 storage/replica_raftstorage.go:605  [s2,r1/90:/M{in-ax}] applied preemptive snapshot in 6ms [clear=1ms batch=0ms entries=3ms commit=1ms]
I170517 07:32:53.702459 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):91): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=91]
I170517 07:32:53.710327 46795 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):91: [(n1,s1):1 (n3,s3):3 (n2,s2):91]
I170517 07:32:53.728937 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):91): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):91, next=92]
I170517 07:32:53.737650 46729 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):91: [(n1,s1):1 (n3,s3):3]
I170517 07:32:53.753444 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 680d8a22 at index 424
I170517 07:32:53.754324 43444 storage/store.go:3152  [s2,r1/91:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:53.756238 43274 storage/store.go:2139  [replicaGC,s2,r1/91:/M{in-ax}] removing replica
I170517 07:32:53.758777 43274 storage/replica.go:684  [replicaGC,s2,r1/91:/M{in-ax}] removed 34 (25+9) keys in 2ms [clear=2ms commit=0ms]
I170517 07:32:53.784145 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 612, log entries: 49, rate-limit: 2.0 MiB/sec, 25ms
I170517 07:32:53.789473 46763 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 424 (id=680d8a22, encoded size=71132, 1 rocksdb batches, 49 log entries)
I170517 07:32:53.805305 46763 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 16ms [clear=1ms batch=0ms entries=12ms commit=2ms]
I170517 07:32:53.808299 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):92): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=92]
I170517 07:32:53.822520 46785 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):92: [(n1,s1):1 (n3,s3):3 (n2,s2):92]
I170517 07:32:53.830455 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):92): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):92, next=93]
I170517 07:32:53.856393 46810 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):92: [(n1,s1):1 (n3,s3):3]
I170517 07:32:53.871597 43444 storage/store.go:3152  [s2,r1/92:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:53.874420 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c61584e0 at index 428
I170517 07:32:53.891000 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 618, log entries: 53, rate-limit: 2.0 MiB/sec, 12ms
I170517 07:32:53.893001 43274 storage/store.go:2139  [replicaGC,s2,r1/92:/M{in-ax}] removing replica
I170517 07:32:53.896968 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):93): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=93]
I170517 07:32:53.901183 43274 storage/replica.go:684  [replicaGC,s2,r1/92:/M{in-ax}] removed 33 (25+8) keys in 8ms [clear=7ms commit=1ms]
I170517 07:32:53.913770 46820 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):93: [(n1,s1):1 (n3,s3):3 (n2,s2):93]
I170517 07:32:53.936358 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):93): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):93, next=94]
I170517 07:32:53.956827 46823 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):93: [(n1,s1):1 (n3,s3):3]
I170517 07:32:53.973154 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c02abdb0 at index 432
E170517 07:32:53.981639 43444 storage/store.go:3150  [s2,r1/93:{-}] unable to add to replica GC queue: replica not initialized
I170517 07:32:54.007107 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 624, log entries: 57, rate-limit: 2.0 MiB/sec, 25ms
I170517 07:32:54.035428 46883 storage/replica_raftstorage.go:597  [s2,r1/93:{-}] applying preemptive snapshot at index 432 (id=c02abdb0, encoded size=77516, 1 rocksdb batches, 57 log entries)
I170517 07:32:54.067989 46883 storage/replica_raftstorage.go:605  [s2,r1/93:/M{in-ax}] applied preemptive snapshot in 32ms [clear=1ms batch=1ms entries=5ms commit=17ms]
I170517 07:32:54.071963 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):94): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=94]
I170517 07:32:54.079723 46901 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):94: [(n1,s1):1 (n3,s3):3 (n2,s2):94]
I170517 07:32:54.096065 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):94): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):94, next=95]
I170517 07:32:54.115584 46870 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):94: [(n1,s1):1 (n3,s3):3]
I170517 07:32:54.127158 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 4be238af at index 436
I170517 07:32:54.131537 43444 storage/store.go:3152  [s2,r1/94:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:54.140072 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 630, log entries: 61, rate-limit: 2.0 MiB/sec, 12ms
I170517 07:32:54.143248 43274 storage/store.go:2139  [replicaGC,s2,r1/94:/M{in-ax}] removing replica
I170517 07:32:54.146131 43274 storage/replica.go:684  [replicaGC,s2,r1/94:/M{in-ax}] removed 33 (25+8) keys in 2ms [clear=2ms commit=1ms]
I170517 07:32:54.148978 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):95): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=95]
I170517 07:32:54.158651 46751 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):95: [(n1,s1):1 (n3,s3):3 (n2,s2):95]
I170517 07:32:54.165713 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):95): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):95, next=96]
W170517 07:32:54.171123 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):94: raft group deleted
I170517 07:32:54.188356 46848 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):95: [(n1,s1):1 (n3,s3):3]
I170517 07:32:54.189478 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 2aa8155e at index 439
I170517 07:32:54.196380 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 844b76f9 at index 440
I170517 07:32:54.200455 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot d805efda at index 440
I170517 07:32:54.203829 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot dae0aa26 at index 440
I170517 07:32:54.208493 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 335b2d77 at index 440
I170517 07:32:54.212276 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 9a7b674d at index 440
I170517 07:32:54.237136 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 09a762ff at index 441
I170517 07:32:54.243508 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):95: kv pairs: 636, log entries: 64, rate-limit: 8.0 MiB/sec, 53ms
I170517 07:32:54.243610 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 60a93e4c at index 441
I170517 07:32:54.247222 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 7a880cbb at index 441
I170517 07:32:54.250623 46874 storage/replica_raftstorage.go:597  [s2,r1/95:{-}] applying Raft snapshot at index 439 (id=2aa8155e, encoded size=83280, 1 rocksdb batches, 64 log entries)
I170517 07:32:54.268273 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 9370860d at index 441
I170517 07:32:54.278766 46874 storage/replica_raftstorage.go:605  [s2,r1/95:/M{in-ax}] applied Raft snapshot in 28ms [clear=7ms batch=0ms entries=15ms commit=3ms]
I170517 07:32:54.280690 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot daab089f at index 441
I170517 07:32:54.283049 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot bf3b7f0c at index 441
I170517 07:32:54.284511 43444 storage/store.go:3152  [s2,r1/95:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:54.309077 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 637, log entries: 66, rate-limit: 2.0 MiB/sec, 25ms
I170517 07:32:54.313203 46951 storage/replica_raftstorage.go:597  [s2,r1/95:/M{in-ax}] applying preemptive snapshot at index 441 (id=bf3b7f0c, encoded size=84390, 1 rocksdb batches, 66 log entries)
I170517 07:32:54.323731 46951 storage/replica_raftstorage.go:605  [s2,r1/95:/M{in-ax}] applied preemptive snapshot in 10ms [clear=2ms batch=0ms entries=5ms commit=3ms]
I170517 07:32:54.324293 43274 storage/store.go:2139  [replicaGC,s2,r1/95:/M{in-ax}] removing replica
I170517 07:32:54.332310 43274 storage/replica.go:684  [replicaGC,s2,r1/95:/M{in-ax}] removed 33 (25+8) keys in 2ms [clear=2ms commit=0ms]
I170517 07:32:54.335481 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):96): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=96]
I170517 07:32:54.356423 46974 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):96: [(n1,s1):1 (n3,s3):3 (n2,s2):96]
I170517 07:32:54.369748 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):96): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):96, next=97]
I170517 07:32:54.378886 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot cb43b9bf at index 446
I170517 07:32:54.389542 47044 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):96: [(n1,s1):1 (n3,s3):3]
I170517 07:32:54.398865 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 9e1ad354 at index 447
I170517 07:32:54.401091 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):96: kv pairs: 645, log entries: 71, rate-limit: 8.0 MiB/sec, 22ms
I170517 07:32:54.402331 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 25da2507 at index 447
I170517 07:32:54.404163 47030 storage/replica_raftstorage.go:597  [s2,r1/96:{-}] applying Raft snapshot at index 446 (id=cb43b9bf, encoded size=87942, 1 rocksdb batches, 71 log entries)
I170517 07:32:54.407339 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 659aabe3 at index 447
I170517 07:32:54.410068 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 22bb3f5b at index 447
I170517 07:32:54.413890 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 8d1af8e4 at index 447
I170517 07:32:54.441041 47030 storage/replica_raftstorage.go:605  [s2,r1/96:/M{in-ax}] applied Raft snapshot in 37ms [clear=1ms batch=0ms entries=30ms commit=2ms]
I170517 07:32:54.446424 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 99131625 at index 447
I170517 07:32:54.455199 43444 storage/store.go:3152  [s2,r1/96:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:54.468783 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 645, log entries: 72, rate-limit: 2.0 MiB/sec, 21ms
I170517 07:32:54.471796 46953 storage/replica_raftstorage.go:597  [s2,r1/96:/M{in-ax}] applying preemptive snapshot at index 447 (id=99131625, encoded size=88562, 1 rocksdb batches, 72 log entries)
I170517 07:32:54.494682 46953 storage/replica_raftstorage.go:605  [s2,r1/96:/M{in-ax}] applied preemptive snapshot in 23ms [clear=2ms batch=0ms entries=10ms commit=8ms]
I170517 07:32:54.495176 43274 storage/store.go:2139  [replicaGC,s2,r1/96:/M{in-ax}] removing replica
I170517 07:32:54.498270 43274 storage/replica.go:684  [replicaGC,s2,r1/96:/M{in-ax}] removed 33 (25+8) keys in 3ms [clear=2ms commit=1ms]
I170517 07:32:54.498380 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):97): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=97]
I170517 07:32:54.517803 47004 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):97: [(n1,s1):1 (n3,s3):3 (n2,s2):97]
I170517 07:32:54.532438 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):97): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):97, next=98]
I170517 07:32:54.548618 47034 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):97: [(n1,s1):1 (n3,s3):3]
I170517 07:32:54.550436 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot cb5b1486 at index 451
I170517 07:32:54.557122 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 7c2b3755 at index 452
I170517 07:32:54.567244 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 82d6efc1 at index 452
I170517 07:32:54.571301 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 3835c0bb at index 452
I170517 07:32:54.578409 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):97: kv pairs: 651, log entries: 76, rate-limit: 8.0 MiB/sec, 26ms
I170517 07:32:54.582002 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 50823f97 at index 452
I170517 07:32:54.584354 47066 storage/replica_raftstorage.go:597  [s2,r1/97:{-}] applying Raft snapshot at index 451 (id=cb5b1486, encoded size=91152, 1 rocksdb batches, 76 log entries)
I170517 07:32:54.585057 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot d0bff34d at index 452
I170517 07:32:54.589191 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 945eac5d at index 452
I170517 07:32:54.591965 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 7fa116b5 at index 452
I170517 07:32:54.594728 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 27e896ef at index 452
I170517 07:32:54.600698 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot f8b5ded6 at index 452
I170517 07:32:54.603426 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 94625025 at index 452
I170517 07:32:54.607878 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 9fe8bbed at index 452
I170517 07:32:54.612682 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot ada70a5d at index 452
I170517 07:32:54.616541 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 0b0acbc3 at index 452
I170517 07:32:54.620527 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 95887c12 at index 452
I170517 07:32:54.621695 47066 storage/replica_raftstorage.go:605  [s2,r1/97:/M{in-ax}] applied Raft snapshot in 37ms [clear=1ms batch=0ms entries=27ms commit=5ms]
I170517 07:32:54.624741 43444 storage/store.go:3152  [s2,r1/97:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:54.634240 43274 storage/store.go:2139  [replicaGC,s2,r1/97:/M{in-ax}] removing replica
I170517 07:32:54.636577 43274 storage/replica.go:684  [replicaGC,s2,r1/97:/M{in-ax}] removed 34 (25+9) keys in 2ms [clear=2ms commit=0ms]
I170517 07:32:54.639425 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 67b3a30e at index 452
I170517 07:32:54.649480 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 651, log entries: 77, rate-limit: 2.0 MiB/sec, 9ms
I170517 07:32:54.651721 47138 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 452 (id=67b3a30e, encoded size=91772, 1 rocksdb batches, 77 log entries)
I170517 07:32:54.672015 47138 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 20ms [clear=0ms batch=0ms entries=18ms commit=1ms]
I170517 07:32:54.675274 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):98): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=98]
I170517 07:32:54.684211 47037 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):98: [(n1,s1):1 (n3,s3):3 (n2,s2):98]
I170517 07:32:54.690755 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):98): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):98, next=99]
I170517 07:32:54.717167 47159 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):98: [(n1,s1):1 (n3,s3):3]
I170517 07:32:54.725446 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 6bf8c89e at index 457
I170517 07:32:54.726160 43444 storage/store.go:3152  [s2,r1/98:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:54.730709 43274 storage/store.go:2139  [replicaGC,s2,r1/98:/M{in-ax}] removing replica
I170517 07:32:54.741813 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot d198c645 at index 457
I170517 07:32:54.745071 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot a746b41d at index 457
I170517 07:32:54.745753 43274 storage/replica.go:684  [replicaGC,s2,r1/98:/M{in-ax}] removed 34 (25+9) keys in 6ms [clear=0ms commit=5ms]
I170517 07:32:54.756866 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 657, log entries: 82, rate-limit: 2.0 MiB/sec, 10ms
I170517 07:32:54.758846 47125 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 457 (id=a746b41d, encoded size=94982, 1 rocksdb batches, 82 log entries)
I170517 07:32:54.768694 47125 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 10ms [clear=0ms batch=0ms entries=8ms commit=1ms]
W170517 07:32:54.769413 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):98: raft group deleted
W170517 07:32:54.769678 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):98: raft group deleted
I170517 07:32:54.776205 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):99): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=99]
I170517 07:32:54.786061 47127 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):99: [(n1,s1):1 (n3,s3):3 (n2,s2):99]
I170517 07:32:54.804877 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):99): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):99, next=100]
I170517 07:32:54.834251 47101 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):99: [(n1,s1):1 (n3,s3):3]
I170517 07:32:54.840618 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 368c5ba4 at index 461
I170517 07:32:54.854040 43444 storage/store.go:3152  [s2,r1/99:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:54.866259 43274 storage/store.go:2139  [replicaGC,s2,r1/99:/M{in-ax}] removing replica
I170517 07:32:54.867734 43274 storage/replica.go:684  [replicaGC,s2,r1/99:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:54.868308 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 663, log entries: 86, rate-limit: 2.0 MiB/sec, 20ms
I170517 07:32:54.870491 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):100): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=100]
I170517 07:32:54.880061 47187 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):100: [(n1,s1):1 (n3,s3):3 (n2,s2):100]
I170517 07:32:54.893857 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):100): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):100, next=101]
I170517 07:32:54.902344 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 8d9d6f20 at index 463
I170517 07:32:54.925049 47018 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):100: [(n1,s1):1 (n3,s3):3]
I170517 07:32:54.930629 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot f9df8ea9 at index 465
I170517 07:32:54.937504 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):100: kv pairs: 666, log entries: 88, rate-limit: 8.0 MiB/sec, 35ms
I170517 07:32:54.943556 47205 storage/replica_raftstorage.go:597  [s2,r1/100:{-}] applying Raft snapshot at index 463 (id=8d9d6f20, encoded size=99802, 1 rocksdb batches, 88 log entries)
I170517 07:32:54.956155 47205 storage/replica_raftstorage.go:605  [s2,r1/100:/M{in-ax}] applied Raft snapshot in 11ms [clear=0ms batch=0ms entries=7ms commit=3ms]
I170517 07:32:54.964511 43444 storage/store.go:3152  [s2,r1/100:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:54.966187 43444 storage/store.go:3152  [s2,r1/100:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:54.967668 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 669, log entries: 90, rate-limit: 2.0 MiB/sec, 20ms
I170517 07:32:54.970714 47133 storage/replica_raftstorage.go:597  [s2,r1/100:/M{in-ax}] applying preemptive snapshot at index 465 (id=f9df8ea9, encoded size=101366, 1 rocksdb batches, 90 log entries)
I170517 07:32:55.005401 47133 storage/replica_raftstorage.go:605  [s2,r1/100:/M{in-ax}] applied preemptive snapshot in 34ms [clear=1ms batch=0ms entries=32ms commit=1ms]
I170517 07:32:55.006150 43274 storage/store.go:2139  [replicaGC,s2,r1/100:/M{in-ax}] removing replica
I170517 07:32:55.007383 43274 storage/replica.go:684  [replicaGC,s2,r1/100:/M{in-ax}] removed 33 (25+8) keys in 1ms [clear=0ms commit=0ms]
I170517 07:32:55.009684 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):101): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=101]
I170517 07:32:55.040598 47220 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):101: [(n1,s1):1 (n3,s3):3 (n2,s2):101]
I170517 07:32:55.070034 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):101): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):101, next=102]
I170517 07:32:55.098051 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 18dcf421 at index 470
I170517 07:32:55.117731 47223 storage/replica_raftstorage.go:597  [s2,r1/101:{-}] applying Raft snapshot at index 470 (id=18dcf421, encoded size=43574, 1 rocksdb batches, 3 log entries)
I170517 07:32:55.121104 47223 storage/replica_raftstorage.go:605  [s2,r1/101:/M{in-ax}] applied Raft snapshot in 3ms [clear=1ms batch=0ms entries=0ms commit=1ms]
I170517 07:32:55.124540 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):101: kv pairs: 674, log entries: 3, rate-limit: 8.0 MiB/sec, 26ms
I170517 07:32:55.126613 47193 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):101: [(n1,s1):1 (n3,s3):3]
I170517 07:32:55.136944 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 47a1a2d5 at index 473
I170517 07:32:55.141878 43444 storage/store.go:3152  [s2,r1/101:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:55.145286 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 678, log entries: 6, rate-limit: 2.0 MiB/sec, 8ms
I170517 07:32:55.149915 47253 storage/replica_raftstorage.go:597  [s2,r1/101:/M{in-ax}] applying preemptive snapshot at index 473 (id=47a1a2d5, encoded size=45628, 1 rocksdb batches, 6 log entries)
I170517 07:32:55.153373 47253 storage/replica_raftstorage.go:605  [s2,r1/101:/M{in-ax}] applied preemptive snapshot in 3ms [clear=1ms batch=0ms entries=1ms commit=1ms]
I170517 07:32:55.154337 43274 storage/store.go:2139  [replicaGC,s2,r1/101:/M{in-ax}] removing replica
I170517 07:32:55.156717 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):102): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=102]
I170517 07:32:55.159104 43274 storage/replica.go:684  [replicaGC,s2,r1/101:/M{in-ax}] removed 33 (25+8) keys in 4ms [clear=1ms commit=3ms]
W170517 07:32:55.168076 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):101: raft group deleted
I170517 07:32:55.177471 47200 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):102: [(n1,s1):1 (n3,s3):3 (n2,s2):102]
I170517 07:32:55.202976 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):102): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, (n2,s2):102, next=103]
I170517 07:32:55.219280 43166 storage/replica_raftstorage.go:416  [raftsnapshot,s1,r1/1:/M{in-ax}] generated Raft snapshot 1d80ff52 at index 476
I170517 07:32:55.226248 47269 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):102: [(n1,s1):1 (n3,s3):3]
I170517 07:32:55.232307 43166 storage/store.go:3345  [raftsnapshot,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):102: kv pairs: 684, log entries: 9, rate-limit: 8.0 MiB/sec, 12ms
I170517 07:32:55.233824 47022 storage/replica_raftstorage.go:597  [s2,r1/102:{-}] applying Raft snapshot at index 476 (id=1d80ff52, encoded size=48200, 1 rocksdb batches, 9 log entries)
I170517 07:32:55.240509 47022 storage/replica_raftstorage.go:605  [s2,r1/102:/M{in-ax}] applied Raft snapshot in 6ms [clear=1ms batch=0ms entries=3ms commit=1ms]
I170517 07:32:55.243390 43075 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot b2c9a6bf at index 477
I170517 07:32:55.251910 43444 storage/store.go:3152  [s2,r1/102:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:55.259522 43075 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 684, log entries: 10, rate-limit: 2.0 MiB/sec, 16ms
I170517 07:32:55.267795 47285 storage/replica_raftstorage.go:597  [s2,r1/102:/M{in-ax}] applying preemptive snapshot at index 477 (id=b2c9a6bf, encoded size=48820, 1 rocksdb batches, 10 log entries)
I170517 07:32:55.273442 47285 storage/replica_raftstorage.go:605  [s2,r1/102:/M{in-ax}] applied preemptive snapshot in 5ms [clear=2ms batch=0ms entries=1ms commit=1ms]
I170517 07:32:55.274467 43274 storage/store.go:2139  [replicaGC,s2,r1/102:/M{in-ax}] removing replica
I170517 07:32:55.277772 43075 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):103): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n3,s3):3, next=103]
I170517 07:32:55.279023 43274 storage/replica.go:684  [replicaGC,s2,r1/102:/M{in-ax}] removed 33 (25+8) keys in 3ms [clear=3ms commit=0ms]
W170517 07:32:55.281678 43353 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):102: raft group deleted
I170517 07:32:55.308030 47238 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):103: [(n1,s1):1 (n3,s3):3 (n2,s2):103]
W170517 07:32:55.416612 43070 storage/raft_transport.go:442  raft transport stream to node 1 failed: rpc error: code = Internal desc = transport is closing
I170517 07:32:55.419000 43016 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:58433->127.0.0.1:48665: use of closed network connection
I170517 07:32:55.419594 43003 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:45309->127.0.0.1:45942: read: connection reset by peer
--- PASS: TestRemovePlaceholderRace (12.09s)
=== RUN   TestReplicaGCRace
I170517 07:32:55.469880 47240 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:55.472403 47240 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:54651" > attrs:<> locality:<>
W170517 07:32:55.493717 47240 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:55.498163 47443 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:54651
I170517 07:32:55.506315 47240 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:32:55.506656 47240 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:59621" > attrs:<> locality:<>
W170517 07:32:55.529434 47240 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:55.544487 47529 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:54651
I170517 07:32:55.550033 47240 storage/store.go:1252  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I170517 07:32:55.552193 47240 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:39018" > attrs:<> locality:<>
I170517 07:32:55.568526 47670 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot dd83f3d4 at index 15
I170517 07:32:55.571721 47670 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:55.572828 47551 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=dd83f3d4, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:55.574704 47551 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:55.577355 47670 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:55.582762 47674 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:55.756862 47698 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:55.866885 47240 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 43b69d4c at index 17
I170517 07:32:55.870069 47240 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:55.871826 47338 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=43b69d4c, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:32:55.873433 47338 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:55.877326 47240 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:55.890865 47684 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:56.180174 47240 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:32:56.188292 47689 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2]
I170517 07:32:56.205247 47240 storage/store.go:2139  [s3,r1/?:/M{in-ax}] removing replica
I170517 07:32:56.205893 47240 storage/replica.go:684  [s3,r1/?:/M{in-ax}] removed 34 (25+9) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:56.207029 47695 storage/raft_transport.go:436  raft transport stream to node 3 established
I170517 07:32:56.209759 47737 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/client_test.go:510
W170517 07:32:56.210119 47735 storage/replica.go:2296  [hb,s1,r1/1:/M{in-ax}] shutdown cancellation after 0.0s of attempting command [txn: 29b9c677], BeginTransaction [/System/NodeLiveness/1,/Min), ConditionalPut [/System/NodeLiveness/1,/Min), EndTransaction [/System/NodeLiveness/1,/Min)
I170517 07:32:56.210622 47262 storage/node_liveness.go:352  [hb] heartbeat result is ambiguous (server shutdown); retrying
W170517 07:32:56.211220 47262 storage/node_liveness.go:253  [hb] failed node liveness heartbeat: node unavailable; try another peer
W170517 07:32:56.237369 47695 storage/raft_transport.go:442  raft transport stream to node 3 failed: EOF
I170517 07:32:56.237464 47527 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:39018->127.0.0.1:56110: use of closed network connection
I170517 07:32:56.240332 47265 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:59621->127.0.0.1:39286: use of closed network connection
I170517 07:32:56.240614 47271 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:54651->127.0.0.1:49218: read: connection reset by peer
--- PASS: TestReplicaGCRace (0.84s)
=== RUN   TestStoreRangeRemoveDead
I170517 07:32:56.288464 47563 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:56.288776 47563 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:36768" > attrs:<> locality:<>
W170517 07:32:56.345909 47563 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:56.353943 47749 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:36768
I170517 07:32:56.371560 47563 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:52067" > attrs:<> locality:<>
W170517 07:32:56.421623 47563 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:56.423294 47886 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:36768
I170517 07:32:56.432904 47563 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:56665" > attrs:<> locality:<>
I170517 07:32:56.458305 48019 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 07900e7c at index 15
I170517 07:32:56.463136 48019 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 4ms
I170517 07:32:56.469764 47726 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=07900e7c, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:56.471133 47726 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:56.473905 48019 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:56.484497 47729 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:56.491919 48024 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot aed32b49 at index 17
I170517 07:32:56.495566 48024 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:56.497035 48101 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=aed32b49, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:32:56.498607 48101 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:56.502290 48024 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:56.506268 48116 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:56.515609 48107 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:56.707626 47563 storage/replica_command.go:3538  [replicate,s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:32:56.720681 48179 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2]
E170517 07:32:56.735617 47563 storage/queue.go:782  [replicate,s1,r1/1:/M{in-ax}] 0 of 0 stores with all attributes matching []; likely not enough nodes in cluster
I170517 07:32:56.737993 48078 storage/store.go:2139  [replicaGC,s3,r1/3:/M{in-ax}] removing replica
I170517 07:32:56.738742 48078 storage/replica.go:684  [replicaGC,s3,r1/3:/M{in-ax}] removed 33 (25+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:56.739843 48170 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/store.go:1453
I170517 07:32:56.772931 47744 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:56665->127.0.0.1:52940: use of closed network connection
I170517 07:32:56.773754 47763 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:56.774339 47938 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:52067->127.0.0.1:52585: use of closed network connection
W170517 07:32:56.774877 48116 storage/raft_transport.go:442  raft transport stream to node 1 failed: rpc error: code = Internal desc = transport is closing
I170517 07:32:56.775026 47766 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:56665: operation was canceled"; Reconnecting to {127.0.0.1:56665 <nil>}
I170517 07:32:56.775203 47766 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
W170517 07:32:56.775989 48108 storage/raft_transport.go:442  raft transport stream to node 3 failed: rpc error: code = Internal desc = transport is closing
--- PASS: TestStoreRangeRemoveDead (0.53s)
=== RUN   TestReplicateRogueRemovedNode
I170517 07:32:56.861350 48029 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:56.861675 48029 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:59012" > attrs:<> locality:<>
W170517 07:32:56.943437 48029 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:56.946173 48326 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:59012
I170517 07:32:56.967575 48029 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:60921" > attrs:<> locality:<>
W170517 07:32:56.985913 48029 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:56.995642 48029 storage/store.go:1252  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I170517 07:32:56.995941 48029 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:42516" > attrs:<> locality:<>
I170517 07:32:57.008929 48317 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:59012
I170517 07:32:57.027107 48446 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 3d9f50a8 at index 15
I170517 07:32:57.036085 48446 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 8ms
I170517 07:32:57.038351 48449 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=3d9f50a8, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:32:57.039755 48449 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:32:57.044849 48446 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:57.051575 48188 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:57.059340 48565 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 03415c80 at index 17
I170517 07:32:57.063306 48565 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:57.064438 48567 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=03415c80, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:32:57.066449 48567 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:57.069398 48565 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:57.080615 48191 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:57.091713 48629 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:57.393766 48599 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
W170517 07:32:57.399740 48541 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:57.400444 48616 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
W170517 07:32:57.400687 48614 storage/raft_transport.go:442  raft transport stream to node 3 failed: store 3 was not found
I170517 07:32:57.411567 48617 storage/raft_transport.go:436  raft transport stream to node 3 established
W170517 07:32:57.412483 48570 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):?: no handler registered for (n3,s3):?
W170517 07:32:57.413140 48619 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):?: store 3 was not found
I170517 07:32:57.417317 48199 storage/raft_transport.go:436  raft transport stream to node 3 established
I170517 07:32:57.420171 48574 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2]
W170517 07:32:57.420327 48572 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:57.423003 48572 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:57.430433 48201 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
I170517 07:32:57.438018 48575 storage/raft_transport.go:436  raft transport stream to node 3 established
W170517 07:32:57.442167 48603 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:32:57.443226 48621 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
I170517 07:32:57.450718 48544 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=4]
I170517 07:32:57.469654 48555 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):2: [(n1,s1):1]
E170517 07:32:57.501774 48193 storage/store.go:3150  [s2,r1/2:/M{in-ax}] unable to add to replica GC queue: queue disabled
I170517 07:32:57.502989 48029 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
I170517 07:32:57.509657 48029 storage/store.go:2139  [replicaGC,s2,r1/2:/M{in-ax}] removing replica
I170517 07:32:57.510297 48029 storage/replica.go:684  [replicaGC,s2,r1/2:/M{in-ax}] removed 34 (26+8) keys in 0ms [clear=0ms commit=0ms]
W170517 07:32:57.516158 48613 storage/store.go:3160  [s1] got error from r1, replica (n2,s2):2: raft group deleted
I170517 07:32:57.638606 48029 storage/client_test.go:1173  test clock advanced to: 864003.600000128,0
I170517 07:32:57.643376 48029 storage/store.go:2139  [replicaGC,s3,r1/3:/M{in-ax}] removing replica
I170517 07:32:57.644064 48029 storage/replica.go:684  [replicaGC,s3,r1/3:/M{in-ax}] removed 34 (26+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:57.645364 48796 util/stop/stopper.go:505  quiescing; tasks left:
3      storage/replica.go:4354
W170517 07:32:57.645701 48734 storage/store.go:1339  [s3,r1/3:/M{in-ax}] could not gossip first range descriptor: [NotLeaseHolderError] r1: replica (n3,s3):3 not lease holder; lease holder unknown
I170517 07:32:57.645989 48796 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/replica.go:4354
W170517 07:32:57.646659 48736 storage/store.go:1339  [s3,r1/3:/M{in-ax}] could not gossip node liveness: [NotLeaseHolderError] r1: replica (n3,s3):3 not lease holder; lease holder unknown
I170517 07:32:57.646832 48796 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/replica.go:4354
W170517 07:32:57.647222 48735 storage/store.go:1339  [s3,r1/3:/M{in-ax}] could not gossip system config: [NotLeaseHolderError] r1: replica (n3,s3):3 not lease holder; lease holder unknown
W170517 07:32:57.697444 48611 storage/raft_transport.go:442  raft transport stream to node 2 failed: EOF
I170517 07:32:57.699104 48450 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:42516->127.0.0.1:37632: use of closed network connection
I170517 07:32:57.699748 48440 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:57.699866 48174 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:60921->127.0.0.1:56186: use of closed network connection
I170517 07:32:57.699961 48149 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:59012->127.0.0.1:56897: use of closed network connection
I170517 07:32:57.700310 48342 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:57.700570 48213 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:32:57.700920 48443 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:42516: getsockopt: connection refused"; Reconnecting to {127.0.0.1:42516 <nil>}
I170517 07:32:57.701316 48345 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:60921: getsockopt: connection refused"; Reconnecting to {127.0.0.1:60921 <nil>}
I170517 07:32:57.702043 48345 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: context canceled
I170517 07:32:57.702429 48443 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: context canceled
--- PASS: TestReplicateRogueRemovedNode (0.91s)
=== RUN   TestReplicateRemovedNodeDisruptiveElection
I170517 07:32:57.729015 48797 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:57.730345 48797 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:39124" > attrs:<> locality:<>
W170517 07:32:57.754322 48797 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:57.764418 48777 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:39124
I170517 07:32:57.779100 48797 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:37530" > attrs:<> locality:<>
W170517 07:32:57.805818 48797 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:57.810926 48990 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:39124
I170517 07:32:57.842094 48797 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:47449" > attrs:<> locality:<>
W170517 07:32:57.863494 48797 gossip/gossip.go:1196  [n4] no incoming or outgoing connections
I170517 07:32:57.866857 49125 gossip/client.go:131  [n4] started gossip client to 127.0.0.1:39124
I170517 07:32:57.899398 48797 gossip/gossip.go:297  [n4] NodeDescriptor set to node_id:4 address:<network_field:"tcp" address_field:"127.0.0.1:49872" > attrs:<> locality:<>
I170517 07:32:57.915741 49224 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 1759c322 at index 16
I170517 07:32:57.920007 49224 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 34, log entries: 6, rate-limit: 2.0 MiB/sec, 4ms
I170517 07:32:57.920256 49268 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 16 (id=1759c322, encoded size=6059, 1 rocksdb batches, 6 log entries)
I170517 07:32:57.922144 49268 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:57.924501 49224 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:57.929964 49229 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:57.935579 49298 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 383a6c6f at index 18
I170517 07:32:57.938904 49298 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 37, log entries: 8, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:57.941700 49108 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 18 (id=383a6c6f, encoded size=7619, 1 rocksdb batches, 8 log entries)
I170517 07:32:57.943923 49108 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:57.952503 49298 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:57.957774 49305 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:57.977764 49110 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:57.990804 49131 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 9a7d80a1 at index 20
I170517 07:32:57.994536 49131 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n4,s4):?: kv pairs: 40, log entries: 10, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:57.999852 49134 storage/replica_raftstorage.go:597  [s4,r1/?:{-}] applying preemptive snapshot at index 20 (id=9a7d80a1, encoded size=9243, 1 rocksdb batches, 10 log entries)
I170517 07:32:58.003590 49134 storage/replica_raftstorage.go:605  [s4,r1/?:/M{in-ax}] applied preemptive snapshot in 4ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:58.008808 49131 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n4,s4):4): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:32:58.019679 49201 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n4,s4):4: [(n1,s1):1 (n2,s2):2 (n3,s3):3 (n4,s4):4]
I170517 07:32:58.397142 49292 storage/replica_command.go:3538  [s2,r1/2:/M{in-ax}] change replicas (REMOVE_REPLICA (n1,s1):1): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, (n4,s4):4, next=5]
I170517 07:32:58.415205 49364 storage/replica.go:2590  [s2,r1/2:/M{in-ax}] proposing REMOVE_REPLICA (n1,s1):1: [(n4,s4):4 (n2,s2):2 (n3,s3):3]
I170517 07:32:58.427841 49304 storage/store.go:3152  [s1,r1/1:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:58.427951 48713 storage/store.go:2139  [replicaGC,s1,r1/1:/M{in-ax}] removing replica
I170517 07:32:58.428646 48713 storage/replica.go:684  [replicaGC,s1,r1/1:/M{in-ax}] removed 34 (26+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:58.442734 49296 storage/raft_transport.go:436  raft transport stream to node 2 established
I170517 07:32:58.510629 48901 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:37530->127.0.0.1:43097: use of closed network connection
I170517 07:32:58.511262 48903 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
W170517 07:32:58.511904 49296 storage/raft_transport.go:442  raft transport stream to node 2 failed: EOF
I170517 07:32:58.512764 49085 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:49872->127.0.0.1:49271: use of closed network connection
I170517 07:32:58.515237 48906 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:37530: operation was canceled"; Reconnecting to {127.0.0.1:37530 <nil>}
I170517 07:32:58.515639 48906 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
W170517 07:32:58.516403 49302 storage/raft_transport.go:442  raft transport stream to node 2 failed: EOF
I170517 07:32:58.517226 49003 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:47449->127.0.0.1:57106: use of closed network connection
--- PASS: TestReplicateRemovedNodeDisruptiveElection (0.82s)
=== RUN   TestReplicaTooOldGC
I170517 07:32:58.586588 49365 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:59402" > attrs:<> locality:<>
W170517 07:32:58.615642 49365 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:32:58.623284 49434 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:59402
I170517 07:32:58.630356 49365 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:59353" > attrs:<> locality:<>
W170517 07:32:58.646909 49365 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:32:58.652531 49650 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:59402
I170517 07:32:58.662282 49365 storage/store.go:1252  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I170517 07:32:58.662560 49365 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:59289" > attrs:<> locality:<>
W170517 07:32:58.678630 49365 gossip/gossip.go:1196  [n4] no incoming or outgoing connections
I170517 07:32:58.680689 49765 gossip/client.go:131  [n4] started gossip client to 127.0.0.1:59402
I170517 07:32:58.702630 49365 gossip/gossip.go:297  [n4] NodeDescriptor set to node_id:4 address:<network_field:"tcp" address_field:"127.0.0.1:40466" > attrs:<> locality:<>
I170517 07:32:58.753431 49742 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 9a47e346 at index 16
I170517 07:32:58.756250 49742 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 34, log entries: 6, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:32:58.759918 49744 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 16 (id=9a47e346, encoded size=6059, 1 rocksdb batches, 6 log entries)
I170517 07:32:58.771495 49744 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 6ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:58.778912 49742 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:32:58.784593 49361 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:32:58.795933 49912 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 2f81a697 at index 18
I170517 07:32:58.800408 49922 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 18 (id=2f81a697, encoded size=7619, 1 rocksdb batches, 8 log entries)
I170517 07:32:58.802029 49922 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:58.803540 49912 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 37, log entries: 8, rate-limit: 2.0 MiB/sec, 6ms
I170517 07:32:58.805422 49912 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:32:58.811227 49868 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:32:58.822325 49894 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:32:58.839173 49918 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 82cf5889 at index 20
I170517 07:32:58.842790 49918 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n4,s4):?: kv pairs: 40, log entries: 10, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:32:58.843884 49896 storage/replica_raftstorage.go:597  [s4,r1/?:{-}] applying preemptive snapshot at index 20 (id=82cf5889, encoded size=9243, 1 rocksdb batches, 10 log entries)
I170517 07:32:58.845656 49896 storage/replica_raftstorage.go:605  [s4,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:32:58.848435 49918 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n4,s4):4): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:32:58.860331 49885 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n4,s4):4: [(n1,s1):1 (n2,s2):2 (n3,s3):3 (n4,s4):4]
I170517 07:32:58.936795 49975 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n4,s4):4): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, (n4,s4):4, next=5]
W170517 07:32:58.942492 49942 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n4,s4):4
W170517 07:32:58.943199 49972 storage/store.go:3156  [s1] raft error: node 4 claims to not contain store 4 for replica (n4,s4):4: store 4 was not found
W170517 07:32:58.943433 49970 storage/raft_transport.go:442  raft transport stream to node 4 failed: store 4 was not found
I170517 07:32:58.949042 49957 storage/raft_transport.go:436  raft transport stream to node 4 established
I170517 07:32:58.952515 49961 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n4,s4):4: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
W170517 07:32:58.954543 49887 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n4,s4):4
W170517 07:32:58.955000 49887 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n4,s4):4
W170517 07:32:58.955573 49959 storage/store.go:3156  [s1] raft error: node 4 claims to not contain store 4 for replica (n4,s4):4: store 4 was not found
I170517 07:32:58.966064 49976 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:32:58.967283 49772 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n4,s4):4
W170517 07:32:58.968824 49770 storage/store.go:3156  [s1] raft error: node 4 claims to not contain store 4 for replica (n4,s4):4: store 4 was not found
I170517 07:32:59.797626 49870 storage/store.go:3152  [s4,r1/4:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:59.798550 49924 storage/store.go:3152  [s4,r1/4:/M{in-ax}] added to replica GC queue (peer suggestion)
I170517 07:32:59.800928 49776 storage/store.go:2139  [replicaGC,s4,r1/4:/M{in-ax}] removing replica
I170517 07:32:59.801778 49776 storage/replica.go:684  [replicaGC,s4,r1/4:/M{in-ax}] removed 35 (27+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:32:59.860938 49397 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:59402->127.0.0.1:49590: use of closed network connection
I170517 07:32:59.863139 49590 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:40466->127.0.0.1:37586: use of closed network connection
I170517 07:32:59.863300 49420 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:59353->127.0.0.1:39604: use of closed network connection
I170517 07:32:59.863460 49625 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:59289->127.0.0.1:58812: use of closed network connection
--- PASS: TestReplicaTooOldGC (1.35s)
=== RUN   TestReplicaLazyLoad
I170517 07:32:59.905587 50102 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:32:59.905943 50102 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:57486" > attrs:<> locality:<>
W170517 07:32:59.908997 50199 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip system config: periodic gossip is disabled
W170517 07:32:59.910404 50200 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip node liveness: periodic gossip is disabled
I170517 07:32:59.994433 50102 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key /Table/50 [r2]
E170517 07:33:00.006028 50177 storage/queue.go:634  [replicate,s1,r1/1:/{Min-Table/50}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:33:00.006503 50177 storage/queue.go:634  [replicate,s1,r2/1:/{Table/50-Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:33:00.008459 50102 util/stop/stopper.go:505  quiescing; tasks left:
1      kv/txn_coord_sender.go:981
W170517 07:33:00.069222 50291 storage/store.go:1339  [s1,r1/1:/{Min-Table/50}] could not gossip system config: periodic gossip is disabled
W170517 07:33:00.069632 50292 storage/store.go:1339  [s1,r1/1:/{Min-Table/50}] could not gossip node liveness: periodic gossip is disabled
W170517 07:33:00.075300 49969 storage/node_liveness.go:253  [hb] failed node liveness heartbeat: node unavailable; try another peer
--- PASS: TestReplicaLazyLoad (0.22s)
=== RUN   TestReplicateReAddAfterDown
I170517 07:33:00.124106 50208 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:56316" > attrs:<> locality:<>
W170517 07:33:00.136146 50208 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:00.156830 50208 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:33:00.157118 50208 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:44142" > attrs:<> locality:<>
I170517 07:33:00.182953 50546 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:56316
W170517 07:33:00.211106 50208 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:33:00.214271 50582 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:56316
I170517 07:33:00.244897 50208 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:43241" > attrs:<> locality:<>
I170517 07:33:00.259861 50454 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 4cf1633b at index 15
I170517 07:33:00.265056 50454 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 5ms
I170517 07:33:00.270481 50303 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=4cf1633b, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:33:00.277027 50303 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 6ms [clear=0ms batch=0ms entries=3ms commit=0ms]
I170517 07:33:00.279652 50454 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:00.285441 50614 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:00.293426 50707 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 2f4c2014 at index 17
I170517 07:33:00.322181 50707 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 28ms
I170517 07:33:00.323644 50456 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=2f4c2014, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:33:00.325301 50456 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:33:00.328221 50707 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:00.332384 50620 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:00.343366 50722 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:33:00.657168 50700 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
W170517 07:33:00.660448 50743 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:33:00.661121 50461 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
W170517 07:33:00.661414 50741 storage/raft_transport.go:442  raft transport stream to node 3 failed: store 3 was not found
I170517 07:33:00.663229 50030 storage/raft_transport.go:436  raft transport stream to node 3 established
W170517 07:33:00.664233 50032 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:33:00.664906 50704 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
I170517 07:33:00.666757 50754 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2]
I170517 07:33:00.668315 50755 storage/raft_transport.go:436  raft transport stream to node 3 established
W170517 07:33:00.670761 50710 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:33:00.671857 50710 storage/raft_transport.go:257  unable to accept Raft message from (n1,s1):1: no handler registered for (n3,s3):3
W170517 07:33:00.672327 50757 storage/store.go:3156  [s1] raft error: node 3 claims to not contain store 3 for replica (n3,s3):3: store 3 was not found
I170517 07:33:00.713882 50867 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 9ed714de at index 23
I170517 07:33:00.719991 50867 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 44, log entries: 13, rate-limit: 2.0 MiB/sec, 5ms
I170517 07:33:00.720681 50712 storage/replica_raftstorage.go:597  [s3,r1/3:/M{in-ax}] applying preemptive snapshot at index 23 (id=9ed714de, encoded size=11133, 1 rocksdb batches, 13 log entries)
I170517 07:33:00.722878 50712 storage/replica_raftstorage.go:605  [s3,r1/3:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:33:00.725107 50867 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):4): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=4]
I170517 07:33:00.739185 50767 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):4: [(n1,s1):1 (n2,s2):2 (n3,s3):4]
I170517 07:33:00.907330 50552 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:43241->127.0.0.1:46092: use of closed network connection
I170517 07:33:00.907708 50364 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
W170517 07:33:00.908215 50457 storage/raft_transport.go:442  raft transport stream to node 2 failed: EOF
W170517 07:33:00.908432 50887 storage/raft_transport.go:442  raft transport stream to node 3 failed: EOF
I170517 07:33:00.909246 50448 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:44142->127.0.0.1:52379: use of closed network connection
W170517 07:33:00.923534 50620 storage/raft_transport.go:442  raft transport stream to node 1 failed: EOF
I170517 07:33:00.923845 50367 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:43241: getsockopt: connection refused"; Reconnecting to {127.0.0.1:43241 <nil>}
I170517 07:33:00.924010 50367 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestReplicateReAddAfterDown (0.84s)
=== RUN   TestLeaseHolderRemoveSelf
I170517 07:33:00.964567 50858 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:00.964904 50858 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:36098" > attrs:<> locality:<>
W170517 07:33:00.986680 50858 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:00.995346 50880 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:36098
I170517 07:33:01.005153 50858 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:52387" > attrs:<> locality:<>
I170517 07:33:01.028694 51124 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 5726e63f at index 14
I170517 07:33:01.031483 51124 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 32, log entries: 4, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:01.032764 50991 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 14 (id=5726e63f, encoded size=5089, 1 rocksdb batches, 4 log entries)
I170517 07:33:01.034525 50991 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=0ms commit=1ms]
I170517 07:33:01.036956 51124 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:01.045010 51089 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:01.153644 51046 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:01.190794 51155 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n1,s1):1): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:01.199384 51142 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n1,s1):1: [(n2,s2):2]
E170517 07:33:01.200089 51142 storage/replica.go:2601  [s1,r1/1:/M{in-ax}] received invalid ChangeReplicasTrigger trigger:<change_type:REMOVE_REPLICA replica:<node_id:1 store_id:1 replica_id:1 > updated_replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:3 >  to remove leaseholder replica {RaftAppliedIndex:17 LeaseAppliedIndex:5 Desc:r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3] Lease:repl=(n1,s1):1 start=0.000000000,0 exp=0.900000123,6 pro=0.000000123,7 TruncatedState:index:10 term:5  GCThreshold:0.000000000,0 Stats:{ContainsEstimates:false LastUpdateNanos:123 IntentAge:0 GCBytesAge:0 LiveBytes:3205 LiveCount:24 KeyBytes:557 KeyCount:24 ValBytes:2718 ValCount:26 IntentBytes:0 IntentCount:0 SysBytes:697 SysCount:9} TxnSpanGCThreshold:0.000000000,0}
I170517 07:33:01.246113 50573 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:52387->127.0.0.1:37996: use of closed network connection
I170517 07:33:01.247381 50748 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:36098->127.0.0.1:44003: use of closed network connection
W170517 07:33:01.247599 51046 storage/raft_transport.go:442  raft transport stream to node 1 failed: EOF
--- PASS: TestLeaseHolderRemoveSelf (0.34s)
=== RUN   TestRemovedReplicaError
I170517 07:33:01.316331 50881 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:01.316639 50881 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:50069" > attrs:<> locality:<>
W170517 07:33:01.346175 50881 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:01.358363 51288 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:50069
I170517 07:33:01.368851 50881 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:33:01.369508 50881 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:51653" > attrs:<> locality:<>
I170517 07:33:01.390650 51145 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot f0e92207 at index 14
I170517 07:33:01.393449 51145 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 32, log entries: 4, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:01.394829 51160 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 14 (id=f0e92207, encoded size=5089, 1 rocksdb batches, 4 log entries)
I170517 07:33:01.396108 51160 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:01.398682 51145 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:01.403763 51404 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:01.513552 51146 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:01.613377 51407 storage/replica_command.go:3538  [s2,r1/2:/M{in-ax}] change replicas (REMOVE_REPLICA (n1,s1):1): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:01.622189 51278 storage/replica.go:2590  [s2,r1/2:/M{in-ax}] proposing REMOVE_REPLICA (n1,s1):1: [(n2,s2):2]
E170517 07:33:01.630736 51208 storage/replica_proposal.go:612  [s1,r1/1:/M{in-ax}] unable to add to replica GC queue: queue disabled
E170517 07:33:01.632209 51167 storage/store.go:3150  [s1,r1/1:/M{in-ax}] unable to add to replica GC queue: queue disabled
I170517 07:33:01.646249 51110 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:50069->127.0.0.1:44769: use of closed network connection
I170517 07:33:01.646480 51303 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:51653->127.0.0.1:58404: use of closed network connection
I170517 07:33:01.646661 51299 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:33:01.648020 51302 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:51653: operation was canceled"; Reconnecting to {127.0.0.1:51653 <nil>}
I170517 07:33:01.648159 51302 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestRemovedReplicaError (0.43s)
=== RUN   TestRemoveRangeWithoutGC
I170517 07:33:01.732204 51458 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:01.732555 51458 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:50376" > attrs:<> locality:<>
W170517 07:33:01.755977 51458 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:01.757983 51135 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:50376
I170517 07:33:01.778431 51458 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:53237" > attrs:<> locality:<>
I170517 07:33:01.794358 51698 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 922f3ff8 at index 14
I170517 07:33:01.797380 51698 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 32, log entries: 4, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:01.800544 51617 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 14 (id=922f3ff8, encoded size=5089, 1 rocksdb batches, 4 log entries)
I170517 07:33:01.801842 51617 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:01.804365 51698 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:01.814085 51677 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:01.923793 51731 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:01.987056 51688 storage/replica_command.go:3538  [s2,r1/2:/M{in-ax}] change replicas (REMOVE_REPLICA (n1,s1):1): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:02.010820 51431 storage/replica.go:2590  [s2,r1/2:/M{in-ax}] proposing REMOVE_REPLICA (n1,s1):1: [(n2,s2):2]
E170517 07:33:02.032043 51533 storage/replica_proposal.go:612  [s1,r1/1:/M{in-ax}] unable to add to replica GC queue: queue disabled
W170517 07:33:02.051563 51574 storage/raft_transport.go:476  no handler found for store 1 in response range_id:1 from_replica:<node_id:2 store_id:2 replica_id:2 > to_replica:<node_id:1 store_id:1 replica_id:1 > union:<error:<message:"sender replica too old, discarding message" transaction_restart:NONE origin_node:0 detail:<replica_too_old:<replica_id:1 > > now:<wall_time:0 logical:0 > > >
E170517 07:33:02.062829 51458 storage/store.go:1188  [n1,s1,r1/?:/M{in-ax}]: unable to add replica to GC queue: queue disabled
I170517 07:33:02.085231 51458 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
I170517 07:33:02.090836 51458 storage/store.go:2139  [replicaGC,s1,r1/?:/M{in-ax}] removing replica
I170517 07:33:02.091480 51458 storage/replica.go:684  [replicaGC,s1,r1/?:/M{in-ax}] removed 32 (24+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:33:02.092374 51694 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/client_test.go:510
W170517 07:33:02.095205 51439 storage/replica.go:2296  [hb,s2,r1/2:/M{in-ax}] shutdown cancellation after 0.0s of attempting command [txn: f4af2637], BeginTransaction [/System/NodeLiveness/1,/Min), ConditionalPut [/System/NodeLiveness/1,/Min), EndTransaction [/System/NodeLiveness/1,/Min)
I170517 07:33:02.095812 51691 storage/node_liveness.go:352  [hb] heartbeat result is ambiguous (server shutdown); retrying
W170517 07:33:02.096542 51691 storage/node_liveness.go:253  [hb] failed node liveness heartbeat: node unavailable; try another peer
I170517 07:33:02.111570 51133 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:53237->127.0.0.1:34689: use of closed network connection
I170517 07:33:02.112129 51562 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:33:02.114766 51565 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:53237: operation was canceled"; Reconnecting to {127.0.0.1:53237 <nil>}
I170517 07:33:02.114922 51565 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestRemoveRangeWithoutGC (0.42s)
=== RUN   TestCheckConsistencyMultiStore
I170517 07:33:02.176865 51739 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:02.183810 51739 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:39281" > attrs:<> locality:<>
W170517 07:33:02.208383 51739 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:02.216449 51739 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:33:02.216657 51739 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:47335" > attrs:<> locality:<>
I170517 07:33:02.219556 51441 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:39281
W170517 07:33:02.324592 51739 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:33:02.334018 52074 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:39281
I170517 07:33:02.341099 51739 storage/store.go:1252  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I170517 07:33:02.341481 51739 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:60550" > attrs:<> locality:<>
I170517 07:33:02.374429 52178 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot ff04e8e4 at index 15
I170517 07:33:02.378170 52178 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:33:02.379523 52180 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=ff04e8e4, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:33:02.381174 52180 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:02.383528 52178 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:02.388934 52196 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:02.405539 52078 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 118bf5f9 at index 17
I170517 07:33:02.408234 52078 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:02.420223 51585 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=118bf5f9, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:33:02.424945 51585 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 4ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:33:02.427730 52078 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:02.435688 52208 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:02.445476 52182 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
W170517 07:33:02.623565 52242 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n1,s1):?
W170517 07:33:02.624342 52226 storage/store.go:3156  [s2] raft error: node 1 claims to not contain store 1 for replica (n1,s1):?: store 1 was not found
W170517 07:33:02.624628 52208 storage/raft_transport.go:442  raft transport stream to node 1 failed: store 1 was not found
W170517 07:33:02.640168 52243 storage/raft_transport.go:442  raft transport stream to node 3 failed: EOF
I170517 07:33:02.641851 51579 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:39281->127.0.0.1:52961: use of closed network connection
I170517 07:33:02.642116 51949 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken read tcp 127.0.0.1:38417->127.0.0.1:47335: read: connection reset by peer.
I170517 07:33:02.642834 51916 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:47335->127.0.0.1:38417: use of closed network connection
I170517 07:33:02.642934 52072 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:60550->127.0.0.1:37763: use of closed network connection
I170517 07:33:02.645243 51952 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:47335: operation was canceled"; Reconnecting to {127.0.0.1:47335 <nil>}
I170517 07:33:02.645529 51952 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestCheckConsistencyMultiStore (0.52s)
=== RUN   TestCheckInconsistent
I170517 07:33:02.693099 52274 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:02.713678 52274 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:39625" > attrs:<> locality:<>
W170517 07:33:02.770583 52274 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:02.777126 52386 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:39625
I170517 07:33:02.797421 52274 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:55699" > attrs:<> locality:<>
W170517 07:33:02.825989 52274 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:33:02.831128 52504 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:39625
I170517 07:33:02.845538 52274 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:55832" > attrs:<> locality:<>
I170517 07:33:02.860612 52611 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 2ac85c37 at index 15
I170517 07:33:02.865064 52611 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 4ms
I170517 07:33:02.866420 52614 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=2ac85c37, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:33:02.867866 52614 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:02.871633 52611 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:02.877430 52367 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:02.885082 52224 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot d0e5b526 at index 17
I170517 07:33:02.887953 52224 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:02.889422 52629 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=d0e5b526, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:33:02.891762 52629 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:33:02.895027 52224 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:02.901913 52645 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:02.910257 52651 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
E170517 07:33:03.091357 52656 storage/stores_server.go:72  consistency check failed on range ID 1: expected checksum 0cb128de2d067e67423aecdbf7a4afd13c32fc3ccd9b8dba4805d43e8327826ffa9f9e38d968dc6139e68c26d3cba22d00f830b7dbd3841ee8f1c30e88418fd8, got 48541dd1e10e456d5ede781cb9b261fb78c86f4a09fb8270b4bc6e0dd18ca558562e3b1181ef6a5ce011f93496fcc763f09d9bd2a70647a125f77452942d03ca
E170517 07:33:03.093460 52676 storage/replica_command.go:2167  [s1,r1/1:/M{in-ax}] replica (n2,s2):2 is inconsistent: expected checksum 0cb128de2d067e67423aecdbf7a4afd13c32fc3ccd9b8dba4805d43e8327826ffa9f9e38d968dc6139e68c26d3cba22d00f830b7dbd3841ee8f1c30e88418fd8, got 48541dd1e10e456d5ede781cb9b261fb78c86f4a09fb8270b4bc6e0dd18ca558562e3b1181ef6a5ce011f93496fcc763f09d9bd2a70647a125f77452942d03ca
E170517 07:33:03.094912 52189 storage/replica_command.go:2188  [s1,r1/1:/M{in-ax}] consistency check failed with 1 inconsistent replicas; fetching details
E170517 07:33:03.112085 52637 storage/stores_server.go:72  consistency check failed on range ID 1: expected checksum 2513c4c48839852d7d80af2635c2f641cd72479f277fc658a63f773c60617fb47563e3bc48b99a9d2e4755f2aed484ec3466a941156cf0e2660d25a45da71f37, got fc750af41b9a4c0e6ba160a3e24bcd66ffaf1ea7564d657a4dc7ed088a9c9b809540e16b997b84497131873c70d2b6eb9d78c519b8f6fbb1b198ae0bfd667377
E170517 07:33:03.118009 52582 storage/replica_command.go:2167  [s1,r1/1:/M{in-ax}] replica (n2,s2):2 is inconsistent: expected checksum 2513c4c48839852d7d80af2635c2f641cd72479f277fc658a63f773c60617fb47563e3bc48b99a9d2e4755f2aed484ec3466a941156cf0e2660d25a45da71f37, got fc750af41b9a4c0e6ba160a3e24bcd66ffaf1ea7564d657a4dc7ed088a9c9b809540e16b997b84497131873c70d2b6eb9d78c519b8f6fbb1b198ae0bfd667377
--- leaseholder
+++ follower
+0.000000123,186 "e"
+  ts:1970-01-01 00:00:00.000000123 +0000 UTC
+  value:����T
+  raw_key:"e" raw_value:000000000154
E170517 07:33:03.118455 52191 storage/replica_command.go:2183  [s1,r1/1:/M{in-ax}] consistency check failed with 1 inconsistent replicas
I170517 07:33:03.118766 52640 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/replica_command.go:2196
1      storage/client_test.go:510
I170517 07:33:03.119045 52640 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/replica_command.go:2196
W170517 07:33:03.146124 52649 storage/raft_transport.go:257  unable to accept Raft message from (n3,s3):?: no handler registered for (n1,s1):?
W170517 07:33:03.146954 52647 storage/store.go:3156  [s3] raft error: node 1 claims to not contain store 1 for replica (n1,s1):?: store 1 was not found
W170517 07:33:03.147215 52645 storage/raft_transport.go:442  raft transport stream to node 1 failed: store 1 was not found
I170517 07:33:03.152837 52584 storage/raft_transport.go:436  raft transport stream to node 1 established
W170517 07:33:03.157470 52588 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n1,s1):?
W170517 07:33:03.158871 52586 storage/raft_transport.go:476  no handler found for store 2 in response range_id:0 from_replica:<node_id:1 store_id:1 replica_id:0 > to_replica:<node_id:2 store_id:2 replica_id:0 > union:<error:<message:"store 1 was not found" transaction_restart:NONE origin_node:0 detail:<store_not_found:<store_id:1 > > now:<wall_time:0 logical:0 > > >
I170517 07:33:03.174666 52476 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:55832->127.0.0.1:52593: read: connection reset by peer
I170517 07:33:03.174892 52276 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:39625->127.0.0.1:59606: use of closed network connection
--- PASS: TestCheckInconsistent (0.53s)
=== RUN   TestTransferRaftLeadership
I170517 07:33:03.210694 52657 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:03.212262 52657 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:52917" > attrs:<> locality:<>
W170517 07:33:03.260228 52657 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:03.289773 52824 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:52917
I170517 07:33:03.309901 52657 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:38659" > attrs:<> locality:<>
W170517 07:33:03.330779 52657 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:33:03.339006 52667 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:52917
I170517 07:33:03.346159 52657 storage/store.go:1252  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I170517 07:33:03.346473 52657 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:56178" > attrs:<> locality:<>
I170517 07:33:03.373212 53040 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r2]
I170517 07:33:03.388205 52682 storage/replica_raftstorage.go:416  [s1,r2/1:{a-/Max}] generated preemptive snapshot e7535d01 at index 11
I170517 07:33:03.391401 52682 storage/store.go:3345  [s1,r2/1:{a-/Max}] streamed snapshot to (n2,s2):?: kv pairs: 27, log entries: 1, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:03.393015 52684 storage/replica_raftstorage.go:597  [s2,r2/?:{-}] applying preemptive snapshot at index 11 (id=e7535d01, encoded size=3537, 1 rocksdb batches, 1 log entries)
I170517 07:33:03.394480 52684 storage/replica_raftstorage.go:605  [s2,r2/?:{a-/Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:03.408080 52682 storage/replica_command.go:3538  [s1,r2/1:{a-/Max}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r2:{a-/Max} [(n1,s1):1, next=2]
W170517 07:33:03.411999 53062 storage/stores.go:218  range not contained in one range: [/Meta2/Max,"a\x00"), but have [/Min,"a")
I170517 07:33:03.432101 52991 storage/replica.go:2590  [s1,r2/1:{a-/Max}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:03.438691 53073 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:03.440955 52686 storage/replica_raftstorage.go:416  [s1,r2/1:{a-/Max}] generated preemptive snapshot 0c4be746 at index 13
I170517 07:33:03.446872 52686 storage/store.go:3345  [s1,r2/1:{a-/Max}] streamed snapshot to (n3,s3):?: kv pairs: 29, log entries: 3, rate-limit: 2.0 MiB/sec, 5ms
I170517 07:33:03.448279 53050 storage/replica_raftstorage.go:597  [s3,r2/?:{-}] applying preemptive snapshot at index 13 (id=0c4be746, encoded size=5048, 1 rocksdb batches, 3 log entries)
I170517 07:33:03.452490 53050 storage/replica_raftstorage.go:605  [s3,r2/?:{a-/Max}] applied preemptive snapshot in 3ms [clear=0ms batch=0ms entries=2ms commit=0ms]
I170517 07:33:03.455314 52686 storage/replica_command.go:3538  [s1,r2/1:{a-/Max}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r2:{a-/Max} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:03.492256 52814 storage/replica.go:2590  [s1,r2/1:{a-/Max}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:33:03.555347 52657 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
I170517 07:33:03.567255 52853 storage/replica_proposal.go:385  [s2,r2/2:{a-/Max}] new range lease repl=(n2,s2):2 start=0.900000123,7 epo=1 pro=1.800000125,5 following repl=(n1,s1):1 start=0.000000000,0 exp=0.900000123,6 pro=0.000000123,7
I170517 07:33:03.591764 52856 storage/replica_proposal.go:385  [s2,r2/2:{a-/Max}] new range lease repl=(n2,s2):2 start=0.900000123,7 epo=1 pro=1.800000125,14 following repl=(n2,s2):2 start=0.900000123,7 epo=1 pro=1.800000125,5
W170517 07:33:03.804207 53073 storage/raft_transport.go:442  raft transport stream to node 1 failed: rpc error: code = Internal desc = transport is closing
--- PASS: TestTransferRaftLeadership (0.65s)
=== RUN   TestFailedPreemptiveSnapshot
I170517 07:33:03.862424 53106 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:03.863014 53106 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:49342" > attrs:<> locality:<>
W170517 07:33:03.886148 53106 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:03.891090 53240 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:49342
I170517 07:33:03.903738 53106 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:45465" > attrs:<> locality:<>
I170517 07:33:03.935862 53087 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot ba2e85a4 at index 14
I170517 07:33:03.952396 53336 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 14 (id=ba2e85a4, encoded size=5089, 1 rocksdb batches, 4 log entries)
I170517 07:33:03.953719 53336 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:03.956624 53087 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 32, log entries: 4, rate-limit: 2.0 MiB/sec, 19ms
I170517 07:33:03.958666 53087 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:03.964462 53380 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:04.157451 53115 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:04.248283 53106 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 797f5527 at index 16
W170517 07:33:04.266255 53115 storage/raft_transport.go:442  raft transport stream to node 1 failed: EOF
I170517 07:33:04.266651 53108 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:49342->127.0.0.1:48466: use of closed network connection
I170517 07:33:04.266876 53127 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:33:04.267261 53092 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:45465->127.0.0.1:41446: use of closed network connection
I170517 07:33:04.267395 53228 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:33:04.267860 53130 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:49342: operation was canceled"; Reconnecting to {127.0.0.1:49342 <nil>}
I170517 07:33:04.268320 53231 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:45465: getsockopt: connection refused"; Reconnecting to {127.0.0.1:45465 <nil>}
I170517 07:33:04.268479 53231 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
I170517 07:33:04.268690 53130 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestFailedPreemptiveSnapshot (0.44s)
=== RUN   TestRaftBlockedReplica
I170517 07:33:04.307995 53365 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:04.308332 53365 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:45156" > attrs:<> locality:<>
W170517 07:33:04.335317 53365 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:04.342779 53096 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:45156
I170517 07:33:04.349169 53365 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:33449" > attrs:<> locality:<>
W170517 07:33:04.367064 53365 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:33:04.375723 53421 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:45156
I170517 07:33:04.380312 53365 storage/store.go:1252  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I170517 07:33:04.380609 53365 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:34847" > attrs:<> locality:<>
I170517 07:33:04.404172 53365 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "b" [r2]
E170517 07:33:04.424601 53479 storage/queue.go:634  [replicate,s1,r1/1:{/Min-b}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:33:04.425189 53479 storage/queue.go:634  [replicate,s1,r2/1:{b-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:33:04.433389 53717 storage/replica_raftstorage.go:416  [s1,r1/1:{/Min-b}] generated preemptive snapshot ddacca84 at index 18
I170517 07:33:04.436790 53740 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 18 (id=ddacca84, encoded size=5315, 1 rocksdb batches, 8 log entries)
I170517 07:33:04.438310 53740 storage/replica_raftstorage.go:605  [s2,r1/?:{/Min-b}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:33:04.438949 53717 storage/store.go:3345  [s1,r1/1:{/Min-b}] streamed snapshot to (n2,s2):?: kv pairs: 19, log entries: 8, rate-limit: 2.0 MiB/sec, 5ms
I170517 07:33:04.440999 53717 storage/replica_command.go:3538  [s1,r1/1:{/Min-b}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:{/Min-b} [(n1,s1):1, next=2]
I170517 07:33:04.446278 53578 storage/replica.go:2590  [s1,r1/1:{/Min-b}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:04.477701 53748 storage/replica_raftstorage.go:416  [s1,r1/1:{/Min-b}] generated preemptive snapshot a0148c04 at index 20
I170517 07:33:04.480663 53748 storage/store.go:3345  [s1,r1/1:{/Min-b}] streamed snapshot to (n3,s3):?: kv pairs: 22, log entries: 10, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:04.482086 53580 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 20 (id=a0148c04, encoded size=6863, 1 rocksdb batches, 10 log entries)
I170517 07:33:04.483887 53580 storage/replica_raftstorage.go:605  [s3,r1/?:{/Min-b}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:33:04.486695 53748 storage/replica_command.go:3538  [s1,r1/1:{/Min-b}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:{/Min-b} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:04.492903 53724 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:04.501511 53616 storage/replica.go:2590  [s1,r1/1:{/Min-b}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:33:04.955234 53373 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:45156->127.0.0.1:44201: use of closed network connection
I170517 07:33:04.956953 52622 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:33449->127.0.0.1:52041: use of closed network connection
W170517 07:33:04.957088 53751 storage/raft_transport.go:442  raft transport stream to node 2 failed: rpc error: code = Internal desc = transport is closing
W170517 07:33:04.957292 53617 storage/raft_transport.go:442  raft transport stream to node 3 failed: EOF
--- PASS: TestRaftBlockedReplica (0.74s)
=== RUN   TestRangeQuiescence
I170517 07:33:05.079038 53798 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
W170517 07:33:05.082009 53874 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip node liveness: periodic gossip is disabled
W170517 07:33:05.082203 53873 storage/store.go:1339  [s1,r1/1:/M{in-ax}] could not gossip system config: periodic gossip is disabled
I170517 07:33:05.089356 53798 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:43059" > attrs:<> locality:<>
W170517 07:33:05.137010 53798 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:05.139164 53896 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:43059
I170517 07:33:05.159631 53798 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:33:05.159948 53798 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:56301" > attrs:<> locality:<>
W170517 07:33:05.201075 53798 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:33:05.205777 53899 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:43059
I170517 07:33:05.219754 53798 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:51640" > attrs:<> locality:<>
I170517 07:33:05.232708 54134 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 4df2bba1 at index 15
I170517 07:33:05.237090 54134 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 4ms
I170517 07:33:05.238348 54146 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=4df2bba1, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:33:05.239760 54146 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:05.242124 54134 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:05.248127 54108 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:05.260138 54162 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 158a7ae3 at index 17
I170517 07:33:05.263001 54162 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:05.268141 54165 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=158a7ae3, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:33:05.271097 54165 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 3ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:33:05.274078 54162 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:05.281351 54122 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:05.294636 54169 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
W170517 07:33:06.108842 54122 storage/raft_transport.go:442  raft transport stream to node 1 failed: rpc error: code = Internal desc = transport is closing
I170517 07:33:06.110671 53910 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:56301->127.0.0.1:45957: use of closed network connection
I170517 07:33:06.112003 53789 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:43059->127.0.0.1:41461: use of closed network connection
W170517 07:33:06.112408 54127 storage/raft_transport.go:442  raft transport stream to node 3 failed: EOF
--- PASS: TestRangeQuiescence (1.11s)
=== RUN   TestInitRaftGroupOnRequest
I170517 07:33:06.146947 54172 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:06.151023 54172 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:35366" > attrs:<> locality:<>
W170517 07:33:06.187096 54172 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:06.188736 54308 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:35366
I170517 07:33:06.197569 54172 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:44978" > attrs:<> locality:<>
I170517 07:33:06.214028 54172 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key /Table/50 [r2]
E170517 07:33:06.224771 54161 storage/queue.go:634  [replicate,s1,r1/1:/{Min-Table/50}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:33:06.225621 54161 storage/queue.go:634  [replicate,s1,r2/1:/{Table/50-Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:33:06.228093 54410 storage/replica_raftstorage.go:416  [s1,r2/1:/{Table/50-Max}] generated preemptive snapshot 5f4bc580 at index 11
I170517 07:33:06.244473 54410 storage/store.go:3345  [s1,r2/1:/{Table/50-Max}] streamed snapshot to (n2,s2):?: kv pairs: 8, log entries: 1, rate-limit: 2.0 MiB/sec, 16ms
I170517 07:33:06.245764 54434 storage/replica_raftstorage.go:597  [s2,r2/?:{-}] applying preemptive snapshot at index 11 (id=5f4bc580, encoded size=460, 1 rocksdb batches, 1 log entries)
I170517 07:33:06.246885 54434 storage/replica_raftstorage.go:605  [s2,r2/?:/{Table/50-Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:06.249356 54410 storage/replica_command.go:3538  [s1,r2/1:/{Table/50-Max}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r2:/{Table/50-Max} [(n1,s1):1, next=2]
W170517 07:33:06.257122 54415 storage/stores.go:218  range not contained in one range: [/Meta2/Max,/Table/50/NULL), but have [/Min,/Table/50)
I170517 07:33:06.269120 54452 storage/replica.go:2590  [s1,r2/1:/{Table/50-Max}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:06.278775 54312 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:06.295082 54172 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/intent_resolver.go:259
2      storage/client_test.go:510
W170517 07:33:06.295308 54419 storage/replica.go:2296  [s1,r2/1:/{Table/50-Max}] shutdown cancellation after 0.0s of attempting command GC [/Table/50,/Table/50/NULL)
W170517 07:33:06.295618 54422 storage/replica.go:2296  [s1,r1/1:/{Min-Table/50}] shutdown cancellation after 0.0s of attempting command ResolveIntent [/Meta2/Max,/Min)
W170517 07:33:06.295790 54466 storage/intent_resolver.go:370  could not GC completed transaction anchored at /Local/Range/"\xba"/RangeDescriptor: result is ambiguous (server shutdown)
I170517 07:33:06.295964 54172 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/intent_resolver.go:259
W170517 07:33:06.296186 54451 storage/intent_resolver.go:309  [n1,s1,r1/1:/{Min-Table/50}]: failed to resolve intents: result is ambiguous (server shutdown)
I170517 07:33:06.296328 54172 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
W170517 07:33:06.300976 54421 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n1,s1):1
W170517 07:33:06.301746 54314 storage/store.go:3156  [s2] raft error: node 1 claims to not contain store 1 for replica (n1,s1):1: store 1 was not found
W170517 07:33:06.302001 54312 storage/raft_transport.go:442  raft transport stream to node 1 failed: store 1 was not found
I170517 07:33:06.440504 54649 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/client_test.go:510
W170517 07:33:06.441508 54647 storage/node_liveness.go:253  [hb] failed node liveness heartbeat: node unavailable; try another peer
W170517 07:33:06.442028 54548 storage/node_liveness.go:253  [hb] failed node liveness heartbeat: node unavailable; try another peer
--- PASS: TestInitRaftGroupOnRequest (0.38s)
=== RUN   TestFailedConfChange
I170517 07:33:06.557992 54651 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:56860" > attrs:<> locality:<>
W170517 07:33:06.577771 54651 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:06.583591 54446 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:56860
I170517 07:33:06.632784 54651 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:53872" > attrs:<> locality:<>
W170517 07:33:06.647951 54651 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:33:06.651137 54474 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:56860
I170517 07:33:06.684529 54651 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:56635" > attrs:<> locality:<>
I170517 07:33:06.701026 55016 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 724b135a at index 15
I170517 07:33:06.707051 55016 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 4ms
I170517 07:33:06.708121 55026 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=724b135a, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:33:06.709639 55026 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:06.712035 55016 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:06.719337 55030 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:06.772080 54832 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:06.803932 55040 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c8ac35c4 at index 17
I170517 07:33:06.813042 55040 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 8ms
I170517 07:33:06.814422 55044 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=c8ac35c4, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:33:06.816010 55044 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:33:06.818841 55040 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:06.826676 54480 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
W170517 07:33:06.936018 55042 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n1,s1):?
W170517 07:33:06.936556 55042 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n1,s1):1
W170517 07:33:06.937128 55037 storage/raft_transport.go:476  no handler found for store 2 in response range_id:0 from_replica:<node_id:1 store_id:1 replica_id:0 > to_replica:<node_id:2 store_id:2 replica_id:0 > union:<error:<message:"store 1 was not found" transaction_restart:NONE origin_node:0 detail:<store_not_found:<store_id:1 > > now:<wall_time:0 logical:0 > > >
W170517 07:33:06.937556 55037 storage/raft_transport.go:476  no handler found for store 2 in response range_id:1 from_replica:<node_id:1 store_id:1 replica_id:1 > to_replica:<node_id:2 store_id:2 replica_id:2 > union:<error:<message:"store 1 was not found" transaction_restart:NONE origin_node:0 detail:<store_not_found:<store_id:1 > > now:<wall_time:0 logical:0 > > >
I170517 07:33:06.941007 54287 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:56860->127.0.0.1:56227: use of closed network connection
I170517 07:33:06.942769 54472 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:56635->127.0.0.1:37353: use of closed network connection
I170517 07:33:06.942914 54774 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:53872->127.0.0.1:55666: use of closed network connection
W170517 07:33:06.943227 54832 storage/raft_transport.go:442  raft transport stream to node 1 failed: rpc error: code = Internal desc = transport is closing
--- PASS: TestFailedConfChange (0.52s)
=== RUN   TestReplicaGCQueueDropReplicaDirect
I170517 07:33:07.056021 54916 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:42268" > attrs:<> locality:<>
W170517 07:33:07.080367 54916 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:07.082843 55209 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:42268
I170517 07:33:07.096379 54916 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:52293" > attrs:<> locality:<>
W170517 07:33:07.109920 54916 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:33:07.112591 55309 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:42268
I170517 07:33:07.138520 54916 storage/store.go:1252  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I170517 07:33:07.138989 54916 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:34094" > attrs:<> locality:<>
I170517 07:33:07.213315 55082 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot b49b8cac at index 15
I170517 07:33:07.215956 55082 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:07.216858 55084 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=b49b8cac, encoded size=6552, 1 rocksdb batches, 5 log entries)
I170517 07:33:07.218284 55084 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:07.221943 55082 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:07.230820 55340 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:07.238513 55433 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 5e7eb689 at index 17
I170517 07:33:07.240965 55433 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:07.242371 55442 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=5e7eb689, encoded size=8689, 1 rocksdb batches, 7 log entries)
I170517 07:33:07.244450 55442 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:33:07.247110 55433 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:07.252483 55325 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:07.266476 55276 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:33:07.417613 55460 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:33:07.426761 55281 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):2: [(n1,s1):1 (n3,s3):3]
I170517 07:33:07.450638 54976 storage/store.go:2139  [replicaGC,s2,r1/2:/M{in-ax}] removing replica
I170517 07:33:07.451916 54976 storage/replica.go:684  [replicaGC,s2,r1/2:/M{in-ax}] removed 33 (25+8) keys in 0ms [clear=0ms commit=0ms]
I170517 07:33:07.477769 55207 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:52293->127.0.0.1:46363: use of closed network connection
I170517 07:33:07.478967 55307 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:34094->127.0.0.1:54813: read: connection reset by peer
--- PASS: TestReplicaGCQueueDropReplicaDirect (0.47s)
=== RUN   TestReplicaGCQueueDropReplicaGCOnScan
I170517 07:33:07.528253 55475 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:07.528609 55475 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:57852" > attrs:<> locality:<>
W170517 07:33:07.574249 55475 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:07.581546 55623 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:57852
I170517 07:33:07.600038 55475 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:50377" > attrs:<> locality:<>
W170517 07:33:07.633700 55475 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:33:07.642819 55536 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:57852
I170517 07:33:07.661377 55475 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:40074" > attrs:<> locality:<>
I170517 07:33:07.685836 55833 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 1e447268 at index 15
I170517 07:33:07.691289 55853 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=1e447268, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:33:07.692623 55853 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:07.694227 55833 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 7ms
I170517 07:33:07.708495 55833 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:07.727106 55856 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:07.747835 55841 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot d7db84c0 at index 17
I170517 07:33:07.753640 55841 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 5ms
I170517 07:33:07.755172 55860 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=d7db84c0, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:33:07.757917 55860 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 3ms [clear=0ms batch=0ms entries=1ms commit=1ms]
I170517 07:33:07.764349 55841 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:07.780967 55893 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:07.802793 55862 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:33:07.964220 55903 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (REMOVE_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:33:07.979102 55923 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA (n2,s2):2: [(n1,s1):1 (n3,s3):3]
E170517 07:33:07.988236 55648 storage/replica_proposal.go:612  [s2,r1/2:/M{in-ax}] unable to add to replica GC queue: queue disabled
E170517 07:33:08.006885 55895 storage/store.go:3150  [s2,r1/2:/M{in-ax}] unable to add to replica GC queue: queue disabled
E170517 07:33:08.007473 55895 storage/store.go:3150  [s2,r1/2:/M{in-ax}] unable to add to replica GC queue: queue disabled
E170517 07:33:08.008072 55895 storage/store.go:3150  [s2,r1/2:/M{in-ax}] unable to add to replica GC queue: queue disabled
I170517 07:33:08.015119 55475 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
I170517 07:33:08.065080 55475 storage/store.go:2139  [replicaGC,s2,r1/2:/M{in-ax}] removing replica
I170517 07:33:08.065785 55475 storage/replica.go:684  [replicaGC,s2,r1/2:/M{in-ax}] removed 33 (25+8) keys in 0ms [clear=0ms commit=0ms]
W170517 07:33:08.077828 55897 storage/raft_transport.go:257  unable to accept Raft message from (n3,s3):3: no handler registered for (n1,s1):1
W170517 07:33:08.078460 55895 storage/store.go:3156  [s3] raft error: node 1 claims to not contain store 1 for replica (n1,s1):1: store 1 was not found
W170517 07:33:08.078732 55893 storage/raft_transport.go:442  raft transport stream to node 1 failed: store 1 was not found
I170517 07:33:08.113308 55621 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:50377->127.0.0.1:58291: use of closed network connection
I170517 07:33:08.113889 55727 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:40074->127.0.0.1:32847: use of closed network connection
I170517 07:33:08.114179 55733 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:33:08.114421 55483 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:33:08.115143 55736 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:40074: getsockopt: connection refused"; Reconnecting to {127.0.0.1:40074 <nil>}
I170517 07:33:08.115258 55486 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:57852: operation was canceled"; Reconnecting to {127.0.0.1:57852 <nil>}
I170517 07:33:08.116426 55486 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
I170517 07:33:08.117160 55468 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:57852->127.0.0.1:49286: use of closed network connection
I170517 07:33:08.117421 55736 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: context canceled
--- PASS: TestReplicaGCQueueDropReplicaGCOnScan (0.63s)
=== RUN   TestRangeCommandClockUpdate
I170517 07:33:08.149072 55868 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:08.149429 55868 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:41570" > attrs:<> locality:<>
W170517 07:33:08.186712 55868 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:08.195200 55423 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:41570
I170517 07:33:08.202478 55868 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:58043" > attrs:<> locality:<>
W170517 07:33:08.218234 55868 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:33:08.219772 55918 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:41570
I170517 07:33:08.255030 55868 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:60686" > attrs:<> locality:<>
I170517 07:33:08.276245 56271 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 80d6109f at index 15
I170517 07:33:08.279526 56271 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:08.280547 56279 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=80d6109f, encoded size=5566, 1 rocksdb batches, 5 log entries)
I170517 07:33:08.282048 56279 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:08.284517 56271 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:08.300647 56283 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:08.313411 56158 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot b3deb0e1 at index 17
I170517 07:33:08.317055 56158 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 3ms
I170517 07:33:08.321242 55505 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=b3deb0e1, encoded size=7113, 1 rocksdb batches, 7 log entries)
I170517 07:33:08.370014 55505 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 47ms [clear=0ms batch=0ms entries=1ms commit=18ms]
I170517 07:33:08.372882 56158 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:08.378424 56286 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:08.388201 56199 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
W170517 07:33:08.550461 56063 util/hlc/hlc.go:214  remote wall time is too far ahead (500ms) to be trustworthy - updating anyway
W170517 07:33:08.552838 56215 util/hlc/hlc.go:214  remote wall time is too far ahead (500ms) to be trustworthy - updating anyway
I170517 07:33:08.574202 55932 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:60686->127.0.0.1:44957: use of closed network connection
I170517 07:33:08.574368 55612 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:58043->127.0.0.1:54953: read: connection reset by peer
W170517 07:33:08.575319 56286 storage/raft_transport.go:442  raft transport stream to node 1 failed: rpc error: code = Internal desc = transport is closing
--- PASS: TestRangeCommandClockUpdate (0.45s)
=== RUN   TestRejectFutureCommand
I170517 07:33:08.641449 56330 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:08.642052 56330 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:41710" > attrs:<> locality:<>
--- PASS: TestRejectFutureCommand (0.20s)
=== RUN   TestTxnPutOutOfOrder
I170517 07:33:08.792832 56201 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:08.808208 56201 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:08.856682 56201 storage/replica_command.go:177  [s1,r1/1:/M{in-ax}] test injecting error: Test
--- PASS: TestTxnPutOutOfOrder (0.16s)
=== RUN   TestRangeLookupUseReverse
I170517 07:33:08.944922 56312 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:08.974904 56312 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "g" [r2]
I170517 07:33:08.986333 56312 storage/replica_command.go:2633  [s1,r1/1:{/Min-g}] initiating a split of this range at key "e" [r3]
I170517 07:33:09.000028 56312 storage/replica_command.go:2633  [s1,r1/1:{/Min-e}] initiating a split of this range at key "c" [r4]
I170517 07:33:09.017021 56312 storage/replica_command.go:2633  [s1,r1/1:{/Min-c}] initiating a split of this range at key "a" [r5]
I170517 07:33:09.032385 56312 util/stop/stopper.go:505  quiescing; tasks left:
1      kv/txn_coord_sender.go:981
--- PASS: TestRangeLookupUseReverse (0.14s)
=== RUN   TestRangeTransferLease
I170517 07:33:09.118224 56666 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:09.118587 56666 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:48744" > attrs:<> locality:<>
W170517 07:33:09.140213 56666 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:09.150230 56458 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:48744
I170517 07:33:09.165489 56666 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:51692" > attrs:<> locality:<>
I170517 07:33:09.185911 56666 storage/client_test.go:406  gossip network initialized
I170517 07:33:09.190584 56887 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot d24706ce at index 15
I170517 07:33:09.194593 56465 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=d24706ce, encoded size=6244, 1 rocksdb batches, 5 log entries)
I170517 07:33:09.195962 56465 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:09.199745 56887 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 8ms
I170517 07:33:09.201580 56887 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:09.209568 56892 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:09.318121 56900 storage/raft_transport.go:436  raft transport stream to node 1 established
=== RUN   TestRangeTransferLease/Transfer
=== RUN   TestRangeTransferLease/TransferWithExtension
=== RUN   TestRangeTransferLease/DrainTransfer
=== RUN   TestRangeTransferLease/DrainTransferWithExtension
I170517 07:33:09.525916 56968 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/store.go:1453
I170517 07:33:09.528182 56969 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/store.go:1453
1      storage/store.go:1044
1      storage/replica_range_lease.go:243
I170517 07:33:09.532075 56969 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/store.go:1044
1      storage/replica_range_lease.go:243
I170517 07:33:09.546745 56969 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/store.go:1044
W170517 07:33:09.563820 56893 storage/raft_transport.go:442  raft transport stream to node 2 failed: EOF
I170517 07:33:09.564027 56790 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:51692->127.0.0.1:43951: use of closed network connection
I170517 07:33:09.565149 56455 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:48744->127.0.0.1:45309: use of closed network connection
W170517 07:33:09.565316 56900 storage/raft_transport.go:442  raft transport stream to node 1 failed: rpc error: code = Internal desc = transport is closing
I170517 07:33:09.565492 56782 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
--- PASS: TestRangeTransferLease (0.49s)
    --- PASS: TestRangeTransferLease/Transfer (0.01s)
    --- PASS: TestRangeTransferLease/TransferWithExtension (0.08s)
    --- PASS: TestRangeTransferLease/DrainTransfer (0.04s)
    --- PASS: TestRangeTransferLease/DrainTransferWithExtension (0.04s)
=== RUN   TestLeaseMetricsOnSplitAndTransfer
I170517 07:33:09.641729 56971 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:09.646388 56971 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:50894" > attrs:<> locality:<>
W170517 07:33:09.669779 56971 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:09.672097 57122 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:50894
I170517 07:33:09.683194 56971 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:36729" > attrs:<> locality:<>
I170517 07:33:09.706824 57217 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c9b9da0b at index 14
I170517 07:33:09.713038 56636 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 14 (id=c9b9da0b, encoded size=5779, 1 rocksdb batches, 4 log entries)
I170517 07:33:09.714263 57217 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 32, log entries: 4, rate-limit: 2.0 MiB/sec, 6ms
I170517 07:33:09.714532 56636 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:09.717104 57217 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:09.727087 57238 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:09.834471 56639 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:09.878253 56971 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r2]
E170517 07:33:09.907097 56975 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a}] range requires a replication change, but lacks a quorum of live replicas (0/2)
E170517 07:33:09.907766 56975 storage/queue.go:634  [replicate,s1,r2/1:{a-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:33:09.937981 56971 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
I170517 07:33:09.956936 57040 storage/replica_proposal.go:385  [s1,r2/1:{a-/Max}] new range lease repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,8 following repl=(n1,s1):1 start=0.000000000,0 exp=0.900000123,6 pro=0.000000123,7
I170517 07:33:10.068670 57063 storage/replica_proposal.go:385  [s1,r2/1:{a-/Max}] new range lease repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,24 following repl=(n1,s1):1 start=0.000000000,0 epo=1 pro=1.800000125,8
I170517 07:33:10.102496 57103 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:36729->127.0.0.1:46691: use of closed network connection
I170517 07:33:10.104456 56994 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:50894->127.0.0.1:35876: use of closed network connection
--- PASS: TestLeaseMetricsOnSplitAndTransfer (0.53s)
=== RUN   TestLeaseNotUsedAfterRestart
I170517 07:33:10.150544 57086 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:10.154358 57086 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:59735" > attrs:<> locality:<>
W170517 07:33:10.214692 57289 storage/node_liveness.go:253  [hb] failed node liveness heartbeat: node unavailable; try another peer
I170517 07:33:10.219135 57246 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:59735->127.0.0.1:48669: use of closed network connection
I170517 07:33:10.219279 57225 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:33:10.219936 57228 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:59735: operation was canceled"; Reconnecting to {127.0.0.1:59735 <nil>}
I170517 07:33:10.220091 57228 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestLeaseNotUsedAfterRestart (0.12s)
=== RUN   TestLeaseExtensionNotBlockedByRead
W170517 07:33:10.293324 57401 server/status/runtime.go:111  Could not parse build timestamp: parsing time "" as "2006/01/02 15:04:05": cannot parse "" as "2006"
I170517 07:33:10.301858 57401 server/config.go:436  1 storage engine initialized
I170517 07:33:10.303124 57401 server/node.go:454  [n?] store [n0,s0] not bootstrapped
I170517 07:33:10.374652 57401 server/node.go:389  [n?] **** cluster 010acf30-7cb1-4b2e-9fe2-2dbcefe61b5b has been created
I170517 07:33:10.374848 57401 server/node.go:390  [n?] **** add additional nodes by specifying --join=127.0.0.1:48569
I170517 07:33:10.401359 57401 storage/store.go:1252  [n1] [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:10.410265 57501 storage/split_queue.go:92  [split,n1,s1,r1/1:/M{in-ax}] splitting at key /System/""
I170517 07:33:10.410653 57401 server/node.go:467  [n1] initialized store [n1,s1]: {Capacity:536870912 Available:536870912 RangeCount:1 LeaseCount:1}
I170517 07:33:10.410968 57401 server/node.go:351  [n1] node ID 1 initialized
I170517 07:33:10.411216 57401 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:48569" > attrs:<> locality:<>
I170517 07:33:10.411784 57401 storage/stores.go:296  [n1] read 0 node addresses from persistent storage
I170517 07:33:10.412128 57401 server/node.go:608  [n1] connecting to gossip network to verify cluster ID...
I170517 07:33:10.412311 57401 server/node.go:633  [n1] node connected via gossip and verified as part of cluster "010acf30-7cb1-4b2e-9fe2-2dbcefe61b5b"
I170517 07:33:10.413378 57401 server/node.go:405  [n1] node=1: started with [[]=] engine(s) and attributes []
I170517 07:33:10.426271 57401 sql/executor.go:349  [n1] creating distSQLPlanner with address {tcp 127.0.0.1:48569}
I170517 07:33:10.432598 57501 storage/replica_command.go:2633  [split,n1,s1,r1/1:/M{in-ax}] initiating a split of this range at key /System/"" [r2]
I170517 07:33:10.480764 57501 storage/split_queue.go:92  [split,n1,s1,r2/1:/{System/-Max}] splitting at key /System/tsd
I170517 07:33:10.480967 57501 storage/replica_command.go:2633  [split,n1,s1,r2/1:/{System/-Max}] initiating a split of this range at key /System/tsd [r3]
I170517 07:33:10.571120 57401 server/server.go:736  [n1] starting https server at 127.0.0.1:58038
I170517 07:33:10.571282 57401 server/server.go:737  [n1] starting grpc/postgres server at 127.0.0.1:48569
I170517 07:33:10.571373 57401 server/server.go:738  [n1] advertising CockroachDB node at 127.0.0.1:48569
I170517 07:33:10.847277 57501 storage/split_queue.go:92  [split,n1,s1,r3/1:/{System/tsd-Max}] splitting at key /System/"tse"
I170517 07:33:10.847496 57501 storage/replica_command.go:2633  [split,n1,s1,r3/1:/{System/tsd-Max}] initiating a split of this range at key /System/"tse" [r4]
I170517 07:33:10.998874 57501 storage/split_queue.go:92  [split,n1,s1,r4/1:/{System/tse-Max}] splitting at key /Table/0/0
I170517 07:33:10.999037 57501 storage/replica_command.go:2633  [split,n1,s1,r4/1:/{System/tse-Max}] initiating a split of this range at key /Table/0 [r5]
I170517 07:33:11.024407 57401 sql/event_log.go:101  [n1] Event: "alter_table", target: 12, info: {TableName:eventlog Statement:ALTER TABLE system.eventlog ALTER COLUMN uniqueid SET DEFAULT uuid_v4() User:node MutationID:0 CascadeDroppedViews:[]}
I170517 07:33:11.120420 57401 sql/lease.go:372  [n1] publish: descID=12 (eventlog) version=2 mtime=2017-05-17 07:33:11.12024967 +0000 UTC
I170517 07:33:11.165902 57501 storage/split_queue.go:92  [split,n1,s1,r5/1:/{Table/0-Max}] splitting at key /Table/11/0
I170517 07:33:11.166079 57501 storage/replica_command.go:2633  [split,n1,s1,r5/1:/{Table/0-Max}] initiating a split of this range at key /Table/11 [r6]
I170517 07:33:11.293732 57501 storage/split_queue.go:92  [split,n1,s1,r6/1:/{Table/11-Max}] splitting at key /Table/12/0
I170517 07:33:11.293921 57501 storage/replica_command.go:2633  [split,n1,s1,r6/1:/{Table/11-Max}] initiating a split of this range at key /Table/12 [r7]
I170517 07:33:11.383458 57401 server/server.go:848  [n1] done ensuring all necessary migrations have run
I170517 07:33:11.383625 57401 server/server.go:850  [n1] serving sql connections
I170517 07:33:11.394955 57501 storage/split_queue.go:92  [split,n1,s1,r7/1:/{Table/12-Max}] splitting at key /Table/13/0
I170517 07:33:11.395132 57501 storage/replica_command.go:2633  [split,n1,s1,r7/1:/{Table/12-Max}] initiating a split of this range at key /Table/13 [r8]
I170517 07:33:11.427337 57910 sql/event_log.go:101  [n1] Event: "node_join", target: 1, info: {Descriptor:{NodeID:1 Address:{NetworkField:tcp AddressField:127.0.0.1:48569} Attrs: Locality:} ClusterID:010acf30-7cb1-4b2e-9fe2-2dbcefe61b5b StartedAt:1495006390412355334 LastUp:1495006390412355334}
I170517 07:33:11.512785 57501 storage/split_queue.go:92  [split,n1,s1,r8/1:/{Table/13-Max}] splitting at key /Table/14/0
I170517 07:33:11.517474 57501 storage/replica_command.go:2633  [split,n1,s1,r8/1:/{Table/13-Max}] initiating a split of this range at key /Table/14 [r9]
I170517 07:33:11.639105 57501 storage/split_queue.go:92  [split,n1,s1,r9/1:/{Table/14-Max}] splitting at key /Table/15/0
I170517 07:33:11.639279 57501 storage/replica_command.go:2633  [split,n1,s1,r9/1:/{Table/14-Max}] initiating a split of this range at key /Table/15 [r10]
I170517 07:33:11.796196 57401 util/stop/stopper.go:505  quiescing; tasks left:
1      server/node.go:854
I170517 07:33:11.798246 57655 kv/transport_race.go:71  transport race promotion: ran 29 iterations on up to 276 requests
--- PASS: TestLeaseExtensionNotBlockedByRead (1.58s)
=== RUN   TestLeaseInfoRequest
--- SKIP: TestLeaseInfoRequest (0.04s)
	client_replica_test.go:1033: #13503
=== RUN   TestErrorHandlingForNonKVCommand
W170517 07:33:11.918239 58007 server/status/runtime.go:111  Could not parse build timestamp: parsing time "" as "2006/01/02 15:04:05": cannot parse "" as "2006"
I170517 07:33:11.925077 58007 server/config.go:436  1 storage engine initialized
I170517 07:33:11.926270 58007 server/node.go:454  [n?] store [n0,s0] not bootstrapped
I170517 07:33:11.971595 58007 server/node.go:389  [n?] **** cluster bab754fe-e38e-4b2a-8260-c8199785fa5e has been created
I170517 07:33:11.979606 58007 server/node.go:390  [n?] **** add additional nodes by specifying --join=127.0.0.1:51064
I170517 07:33:11.998232 58007 storage/store.go:1252  [n1] [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:11.998380 58007 server/node.go:467  [n1] initialized store [n1,s1]: {Capacity:536870912 Available:536870912 RangeCount:1 LeaseCount:0}
I170517 07:33:11.998547 58007 server/node.go:351  [n1] node ID 1 initialized
I170517 07:33:11.998771 58007 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:51064" > attrs:<> locality:<>
I170517 07:33:11.999609 58007 storage/stores.go:296  [n1] read 0 node addresses from persistent storage
I170517 07:33:11.999784 58007 server/node.go:608  [n1] connecting to gossip network to verify cluster ID...
I170517 07:33:12.002519 58007 server/node.go:633  [n1] node connected via gossip and verified as part of cluster "bab754fe-e38e-4b2a-8260-c8199785fa5e"
I170517 07:33:12.002877 58007 server/node.go:405  [n1] node=1: started with [[]=] engine(s) and attributes []
I170517 07:33:12.016288 58245 storage/split_queue.go:92  [split,n1,s1,r1/1:/M{in-ax}] splitting at key /System/""
I170517 07:33:12.016675 58007 sql/executor.go:349  [n1] creating distSQLPlanner with address {tcp 127.0.0.1:51064}
I170517 07:33:12.024885 58245 storage/replica_command.go:2633  [split,n1,s1,r1/1:/M{in-ax}] initiating a split of this range at key /System/"" [r2]
I170517 07:33:12.071886 58245 storage/split_queue.go:92  [split,n1,s1,r2/1:/{System/-Max}] splitting at key /System/tsd
I170517 07:33:12.072086 58245 storage/replica_command.go:2633  [split,n1,s1,r2/1:/{System/-Max}] initiating a split of this range at key /System/tsd [r3]
I170517 07:33:12.128041 58007 server/server.go:736  [n1] starting https server at 127.0.0.1:48074
I170517 07:33:12.138742 58007 server/server.go:737  [n1] starting grpc/postgres server at 127.0.0.1:51064
I170517 07:33:12.138891 58007 server/server.go:738  [n1] advertising CockroachDB node at 127.0.0.1:51064
I170517 07:33:12.164500 58245 storage/split_queue.go:92  [split,n1,s1,r3/1:/{System/tsd-Max}] splitting at key /System/"tse"
I170517 07:33:12.164726 58245 storage/replica_command.go:2633  [split,n1,s1,r3/1:/{System/tsd-Max}] initiating a split of this range at key /System/"tse" [r4]
I170517 07:33:12.376034 58245 storage/split_queue.go:92  [split,n1,s1,r4/1:/{System/tse-Max}] splitting at key /Table/0/0
I170517 07:33:12.376221 58245 storage/replica_command.go:2633  [split,n1,s1,r4/1:/{System/tse-Max}] initiating a split of this range at key /Table/0 [r5]
I170517 07:33:12.411286 58007 sql/event_log.go:101  [n1] Event: "alter_table", target: 12, info: {TableName:eventlog Statement:ALTER TABLE system.eventlog ALTER COLUMN uniqueid SET DEFAULT uuid_v4() User:node MutationID:0 CascadeDroppedViews:[]}
E170517 07:33:12.427458 58221 storage/replica_proposal.go:458  [n1,s1,r4/1:/{System/tse-Max}] could not load SystemConfig span: must retry later due to intent on SystemConfigSpan
I170517 07:33:12.441100 58245 storage/split_queue.go:92  [split,n1,s1,r5/1:/{Table/0-Max}] splitting at key /Table/11/0
I170517 07:33:12.444786 58245 storage/replica_command.go:2633  [split,n1,s1,r5/1:/{Table/0-Max}] initiating a split of this range at key /Table/11 [r6]
I170517 07:33:12.461754 58007 sql/lease.go:372  [n1] publish: descID=12 (eventlog) version=2 mtime=2017-05-17 07:33:12.461653532 +0000 UTC
I170517 07:33:12.518555 58245 storage/split_queue.go:92  [split,n1,s1,r6/1:/{Table/11-Max}] splitting at key /Table/12/0
I170517 07:33:12.518769 58245 storage/replica_command.go:2633  [split,n1,s1,r6/1:/{Table/11-Max}] initiating a split of this range at key /Table/12 [r7]
I170517 07:33:12.604210 58245 storage/split_queue.go:92  [split,n1,s1,r7/1:/{Table/12-Max}] splitting at key /Table/13/0
I170517 07:33:12.604382 58245 storage/replica_command.go:2633  [split,n1,s1,r7/1:/{Table/12-Max}] initiating a split of this range at key /Table/13 [r8]
I170517 07:33:12.659542 58007 server/server.go:848  [n1] done ensuring all necessary migrations have run
I170517 07:33:12.659663 58007 server/server.go:850  [n1] serving sql connections
I170517 07:33:12.741764 58245 storage/split_queue.go:92  [split,n1,s1,r8/1:/{Table/13-Max}] splitting at key /Table/14/0
I170517 07:33:12.741968 58245 storage/replica_command.go:2633  [split,n1,s1,r8/1:/{Table/13-Max}] initiating a split of this range at key /Table/14 [r9]
I170517 07:33:12.745969 58461 sql/event_log.go:101  [n1] Event: "node_join", target: 1, info: {Descriptor:{NodeID:1 Address:{NetworkField:tcp AddressField:127.0.0.1:51064} Attrs: Locality:} ClusterID:bab754fe-e38e-4b2a-8260-c8199785fa5e StartedAt:1495006392002589244 LastUp:1495006392002589244}
I170517 07:33:12.912692 58245 storage/split_queue.go:92  [split,n1,s1,r9/1:/{Table/14-Max}] splitting at key /Table/15/0
I170517 07:33:12.912920 58245 storage/replica_command.go:2633  [split,n1,s1,r9/1:/{Table/14-Max}] initiating a split of this range at key /Table/15 [r10]
I170517 07:33:13.100781 58606 storage/replica_command.go:177  [n1,s1,r4/1:/{System/tse-Table/0}] test injecting error: storage/client_replica_test.go:1129: injected error
I170517 07:33:13.101477 58007 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
1      server/node.go:854
W170517 07:33:13.103285 58670 storage/replica.go:2296  [n1,s1,r8/1:/Table/1{3-4}] shutdown cancellation after 0.0s of attempting command Noop, Noop, ResolveIntent [/Table/13/1/2017-05-17T07:33:12.912992Z/245550261666775040/0,/Min), ResolveIntent [/Table/13/1/2017-05-17T07:33:12.912992Z/245550261666775040/2/1,/Min), ResolveIntent [/Table/13/1/2017-05-17T07:33:12.912992Z/245550261666775040/3/1,/Min), ResolveIntent [/Table/13/1/2017-05-17T07:33:12.912992Z/245550261666775040/4/1,/Min), ResolveIntent [/Table/13/1/2017-05-17T07:33:12.912992Z/245550261666775040/5/1,/Min), ResolveIntent [/Table/13/1/2017-05-17T07:33:12.912992Z/245550261666775040/6/1,/Min), ResolveIntent [/Table/13/1/2017-05-17T07:33:12.98388Z/245550261987639296/0,/Min), ResolveIntent [/Table/13/1/2017-05-17T07:33:12.98388Z/245550261987639296/2/1,/Min), ResolveIntent [/Table/13/1/2017-05-17T07:33:12.98388Z/245550261987639296/3/1,/Min), ResolveIntent [/Table/13/1/2017-05-17T07:33:12.98388Z/245550261987639296/4/1,/Min), ResolveIntent [/Table/13/1/2017-05-17T07:33:12.98388Z/245550261987639296/5/1,/Min), ResolveIntent [/Table/13/1/2017-05-17T07:33:12.98388Z/245550261987639296/6/1,/Min)
I170517 07:33:13.103883 58007 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
W170517 07:33:13.104235 58559 storage/intent_resolver.go:327  [n1,s1,r9/1:/Table/1{4-5}]: failed to resolve intents: result is ambiguous (server shutdown)
I170517 07:33:13.138048 58307 kv/transport_race.go:71  transport race promotion: ran 45 iterations on up to 254 requests
--- PASS: TestErrorHandlingForNonKVCommand (1.29s)
=== RUN   TestRangeInfo
I170517 07:33:13.177294 58505 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:42929" > attrs:<> locality:<>
W170517 07:33:13.198298 58505 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:13.200982 58646 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:42929
I170517 07:33:13.210107 58505 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:33:13.210413 58505 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:55549" > attrs:<> locality:<>
I170517 07:33:13.228047 58881 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 35fc0c93 at index 14
I170517 07:33:13.230423 58881 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 32, log entries: 4, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:13.231549 58854 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 14 (id=35fc0c93, encoded size=5089, 1 rocksdb batches, 4 log entries)
I170517 07:33:13.233310 58854 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:13.236488 58881 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:13.263236 58918 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:13.411100 58919 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:13.555397 58505 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r2]
E170517 07:33:13.589975 58759 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a}] range requires a replication change, but lacks a quorum of live replicas (0/2)
E170517 07:33:13.590517 58759 storage/queue.go:634  [replicate,s1,r2/1:{a-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/2)
W170517 07:33:13.619909 58887 storage/stores.go:218  range not contained in one range: [/System/Max,/Max), but have [/Min,"a")
I170517 07:33:13.663927 58822 storage/replica_proposal.go:385  [s2,r2/2:{a-/Max}] new range lease repl=(n2,s2):2 start=0.000000123,211 epo=1 pro=0.000000123,212 following repl=(n1,s1):1 start=0.000000000,0 exp=0.900000123,5 pro=0.000000123,6
W170517 07:33:13.698852 58946 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n1,s1):1
W170517 07:33:13.699779 58921 storage/store.go:3156  [s2] raft error: node 1 claims to not contain store 1 for replica (n1,s1):1: store 1 was not found
W170517 07:33:13.700067 58919 storage/raft_transport.go:442  raft transport stream to node 1 failed: store 1 was not found
--- PASS: TestRangeInfo (0.58s)
=== RUN   TestCampaignOnLazyRaftGroupInitialization
--- SKIP: TestCampaignOnLazyRaftGroupInitialization (0.02s)
	client_replica_test.go:1322: this test is flaky on the 5m initial stress due to errant Raft messages initializing the Raft group
=== RUN   TestDrainRangeRejection
I170517 07:33:13.788679 58925 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:13.789015 58925 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:44370" > attrs:<> locality:<>
W170517 07:33:13.850318 58925 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:13.860774 58969 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:44370
I170517 07:33:13.917337 58925 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:51069" > attrs:<> locality:<>
I170517 07:33:13.928781 58925 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot c391d4df at index 14
I170517 07:33:13.940481 59087 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:51069->127.0.0.1:33673: use of closed network connection
I170517 07:33:13.940650 59088 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:33:13.941891 59091 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:51069: operation was canceled"; Reconnecting to {127.0.0.1:51069 <nil>}
I170517 07:33:13.942038 59091 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestDrainRangeRejection (0.23s)
=== RUN   TestStoreRangeSplitAtIllegalKeys
I170517 07:33:13.997191 58959 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:14.036673 58959 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
--- PASS: TestStoreRangeSplitAtIllegalKeys (0.15s)
=== RUN   TestStoreRangeSplitAtTablePrefix
I170517 07:33:14.133488 59038 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:14.166514 59038 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key /Table/50 [r2]
W170517 07:33:14.220750 59038 gossip/gossip.go:842  [n1] raw gossip callback registered on system-db, consider using RegisterSystemConfigChannel
--- PASS: TestStoreRangeSplitAtTablePrefix (0.12s)
=== RUN   TestStoreRangeSplitInsideRow
I170517 07:33:14.250060 59398 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:14.264019 59398 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:14.293360 59398 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key /Table/50/1/1/"a" [r2]
I170517 07:33:14.311310 59398 util/stop/stopper.go:505  quiescing; tasks left:
1      kv/txn_coord_sender.go:981
--- PASS: TestStoreRangeSplitInsideRow (0.14s)
=== RUN   TestStoreRangeSplitIntents
I170517 07:33:14.391039 59465 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:14.408873 59465 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:14.443055 59465 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "m" [r2]
I170517 07:33:14.468352 59465 util/stop/stopper.go:505  quiescing; tasks left:
1      kv/txn_coord_sender.go:981
--- PASS: TestStoreRangeSplitIntents (0.16s)
=== RUN   TestStoreRangeSplitAtRangeBounds
I170517 07:33:14.551114 59523 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:14.568596 59523 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:14.599029 59523 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r2]
--- PASS: TestStoreRangeSplitAtRangeBounds (0.10s)
=== RUN   TestStoreRangeSplitConcurrent
I170517 07:33:14.656999 59717 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:14.715712 59828 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r6]
I170517 07:33:14.715878 59826 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r4]
I170517 07:33:14.716885 59831 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r9]
I170517 07:33:14.723951 59827 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r5]
I170517 07:33:14.725315 59425 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r3]
I170517 07:33:14.725513 59832 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r10]
I170517 07:33:14.729304 59829 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r7]
I170517 07:33:14.729520 59830 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r8]
I170517 07:33:14.729677 59424 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r2]
I170517 07:33:14.738148 59833 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r11]
--- PASS: TestStoreRangeSplitConcurrent (0.32s)
=== RUN   TestStoreRangeSplitIdempotency
I170517 07:33:14.975316 59244 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:14.992277 59244 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:15.032697 59244 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "m" [r2]
--- PASS: TestStoreRangeSplitIdempotency (0.19s)
=== RUN   TestStoreRangeSplitStats
I170517 07:33:15.165918 59712 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:15.185083 59712 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:15.199190 59712 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key /Table/50 [r2]
I170517 07:33:15.487425 59712 storage/replica_command.go:2633  [s1,r2/1:/{Table/50-Max}] initiating a split of this range at key /Table/50/0/"Z" [r3]
I170517 07:33:15.522674 59712 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
W170517 07:33:15.523874 60022 storage/replica.go:2296  [s1,r1/1:/{Min-Table/50}] shutdown cancellation after 0.0s of attempting command ResolveIntent [/Meta2/Table/50/0/"Z",/Min), ResolveIntent [/Meta2/Max,/Min)
W170517 07:33:15.524199 60022 storage/intent_resolver.go:327  [n1,s1,r2/1:/Table/50{-/0/"Z"}]: failed to resolve intents: result is ambiguous (server shutdown)
--- PASS: TestStoreRangeSplitStats (0.42s)
=== RUN   TestStoreRangeSplitStatsWithMerges
I170517 07:33:15.591722 60023 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:15.620655 60023 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key /Table/50 [r2]
I170517 07:33:16.214982 60023 storage/replica_command.go:2633  [s1,r2/1:/{Table/50-Max}] initiating a split of this range at key /Table/50/0/360000000000000 [r3]
W170517 07:33:16.219097 60023 storage/stores.go:218  range not contained in one range: [/Meta2/Table/50/0/360000000000000,/Table/50/0/360000000000000/NULL), but have [/Min,/Table/50)
I170517 07:33:16.295053 60023 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/intent_resolver.go:259
W170517 07:33:16.295693 60122 storage/intent_resolver.go:370  could not GC completed transaction anchored at /Local/Range/"\xba"/RangeDescriptor: failed to send RPC: sending to all 1 replicas failed; last error: node unavailable; try another peer
W170517 07:33:16.296834 60121 storage/intent_resolver.go:309  [n1,s1,r1/1:/{Min-Table/50}]: failed to resolve intents: node unavailable; try another peer
I170517 07:33:16.296991 60023 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
--- PASS: TestStoreRangeSplitStatsWithMerges (0.73s)
=== RUN   TestStoreZoneUpdateAndRangeSplit
I170517 07:33:16.321816 60108 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:16.343781 60108 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:16.404948 60138 storage/split_queue.go:92  [split,s1,r1/1:/M{in-ax}] splitting at key /System/""
I170517 07:33:16.415503 60138 storage/replica_command.go:2633  [split,s1,r1/1:/M{in-ax}] initiating a split of this range at key /System/"" [r2]
I170517 07:33:16.430137 60138 storage/split_queue.go:92  [split,s1,r2/1:/{System/-Max}] splitting at key /System/tsd
I170517 07:33:16.430347 60138 storage/replica_command.go:2633  [split,s1,r2/1:/{System/-Max}] initiating a split of this range at key /System/tsd [r3]
W170517 07:33:16.435166 60138 storage/stores.go:218  range not contained in one range: [/Meta2/System/tsd,/System/tsd/NULL), but have [/Min,/System/"")
I170517 07:33:16.457367 60138 storage/split_queue.go:92  [split,s1,r3/1:/{System/tsd-Max}] splitting at key /System/"tse"
I170517 07:33:16.457585 60138 storage/replica_command.go:2633  [split,s1,r3/1:/{System/tsd-Max}] initiating a split of this range at key /System/"tse" [r4]
I170517 07:33:16.519386 60138 storage/split_queue.go:92  [split,s1,r4/1:/{System/tse-Max}] splitting at key /Table/0/0
I170517 07:33:16.519533 60138 storage/replica_command.go:2633  [split,s1,r4/1:/{System/tse-Max}] initiating a split of this range at key /Table/0 [r5]
I170517 07:33:16.539861 60138 storage/split_queue.go:92  [split,s1,r5/1:/{Table/0-Max}] splitting at key /Table/50/0
I170517 07:33:16.540027 60138 storage/replica_command.go:2633  [split,s1,r5/1:/{Table/0-Max}] initiating a split of this range at key /Table/50 [r6]
I170517 07:33:17.883505 60138 storage/split_queue.go:111  [split,s1,r6/1:/{Table/50-Max}] splitting size=65585 max=65536
I170517 07:33:17.893805 60138 storage/replica_command.go:2633  [split,s1,r6/1:/{Table/50-Max}] initiating a split of this range at key /Table/50/"YsaXTYxLDwuncKjSMuLeoweuOMRPERBjlcscGQjFxqrXtNxYwXdRtnKcOMNBmwvewObjgoJlqotZOkRuCgdKJPDMsdCltBNtbyWw" [r7]
I170517 07:33:17.924067 60108 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/queue.go:523
1      storage/store.go:1453
1      kv/txn_coord_sender.go:981
I170517 07:33:17.924397 60108 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/queue.go:523
1      storage/store.go:1453
W170517 07:33:17.925498 59937 gossip/infostore.go:303  [n1] node unavailable; try another peer
I170517 07:33:17.925757 60108 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/queue.go:523
W170517 07:33:17.926059 60138 storage/intent_resolver.go:266  failed to resolve intents: node unavailable; try another peer
I170517 07:33:17.926811 60108 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/queue.go:523
--- PASS: TestStoreZoneUpdateAndRangeSplit (1.65s)
=== RUN   TestStoreRangeSplitWithMaxBytesUpdate
I170517 07:33:17.986359 60215 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:18.013379 60215 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:18.036162 60233 storage/split_queue.go:92  [split,s1,r1/1:/M{in-ax}] splitting at key /System/""
I170517 07:33:18.059696 60233 storage/replica_command.go:2633  [split,s1,r1/1:/M{in-ax}] initiating a split of this range at key /System/"" [r2]
I170517 07:33:18.089951 60233 storage/split_queue.go:92  [split,s1,r2/1:/{System/-Max}] splitting at key /System/tsd
I170517 07:33:18.090135 60233 storage/replica_command.go:2633  [split,s1,r2/1:/{System/-Max}] initiating a split of this range at key /System/tsd [r3]
W170517 07:33:18.095398 60233 storage/stores.go:218  range not contained in one range: [/Meta2/System/tsd,/System/tsd/NULL), but have [/Min,/System/"")
I170517 07:33:18.155984 60233 storage/split_queue.go:92  [split,s1,r3/1:/{System/tsd-Max}] splitting at key /System/"tse"
I170517 07:33:18.156347 60233 storage/replica_command.go:2633  [split,s1,r3/1:/{System/tsd-Max}] initiating a split of this range at key /System/"tse" [r4]
I170517 07:33:18.211105 60233 storage/split_queue.go:92  [split,s1,r4/1:/{System/tse-Max}] splitting at key /Table/0/0
I170517 07:33:18.211298 60233 storage/replica_command.go:2633  [split,s1,r4/1:/{System/tse-Max}] initiating a split of this range at key /Table/0 [r5]
I170517 07:33:18.234928 60233 storage/split_queue.go:92  [split,s1,r5/1:/{Table/0-Max}] splitting at key /Table/50/0
I170517 07:33:18.235078 60233 storage/replica_command.go:2633  [split,s1,r5/1:/{Table/0-Max}] initiating a split of this range at key /Table/50 [r6]
I170517 07:33:18.316845 60215 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/queue.go:523
--- PASS: TestStoreRangeSplitWithMaxBytesUpdate (0.39s)
=== RUN   TestStoreRangeSystemSplits
I170517 07:33:18.371127 60403 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:18.395624 60403 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:18.403748 60482 storage/split_queue.go:92  [split,s1,r1/1:/M{in-ax}] splitting at key /System/""
I170517 07:33:18.413648 60482 storage/replica_command.go:2633  [split,s1,r1/1:/M{in-ax}] initiating a split of this range at key /System/"" [r2]
E170517 07:33:18.440817 60416 storage/replica_proposal.go:458  [split,s1,r1/1:/M{in-ax}] could not load SystemConfig span: must retry later due to intent on SystemConfigSpan
I170517 07:33:18.445494 60482 storage/split_queue.go:92  [split,s1,r2/1:/{System/-Max}] splitting at key /System/tsd
I170517 07:33:18.445696 60482 storage/replica_command.go:2633  [split,s1,r2/1:/{System/-Max}] initiating a split of this range at key /System/tsd [r3]
E170517 07:33:18.574175 60430 storage/replica_proposal.go:458  [split,s1,r2/1:/{System/-Max}] could not load SystemConfig span: must retry later due to intent on SystemConfigSpan
I170517 07:33:18.612062 60482 storage/split_queue.go:92  [split,s1,r3/1:/{System/tsd-Max}] splitting at key /System/"tse"
I170517 07:33:18.612285 60482 storage/replica_command.go:2633  [split,s1,r3/1:/{System/tsd-Max}] initiating a split of this range at key /System/"tse" [r4]
E170517 07:33:18.654565 60443 storage/replica_proposal.go:458  [split,s1,r3/1:/{System/tsd-Max}] could not load SystemConfig span: must retry later due to intent on SystemConfigSpan
I170517 07:33:18.687664 60482 storage/split_queue.go:92  [split,s1,r4/1:/{System/tse-Max}] splitting at key /Table/0/0
I170517 07:33:18.687848 60482 storage/replica_command.go:2633  [split,s1,r4/1:/{System/tse-Max}] initiating a split of this range at key /Table/0 [r5]
E170517 07:33:18.719476 60457 storage/replica_proposal.go:458  [split,s1,r4/1:/{System/tse-Max}] could not load SystemConfig span: must retry later due to intent on SystemConfigSpan
I170517 07:33:18.741061 60482 storage/split_queue.go:92  [split,s1,r5/1:/{Table/0-Max}] splitting at key /Table/11/0
I170517 07:33:18.741578 60482 storage/replica_command.go:2633  [split,s1,r5/1:/{Table/0-Max}] initiating a split of this range at key /Table/11 [r6]
I170517 07:33:18.893327 60482 storage/split_queue.go:92  [split,s1,r6/1:/{Table/11-Max}] splitting at key /Table/12/0
I170517 07:33:18.893519 60482 storage/replica_command.go:2633  [split,s1,r6/1:/{Table/11-Max}] initiating a split of this range at key /Table/12 [r7]
I170517 07:33:18.985082 60482 storage/split_queue.go:92  [split,s1,r7/1:/{Table/12-Max}] splitting at key /Table/13/0
I170517 07:33:18.985275 60482 storage/replica_command.go:2633  [split,s1,r7/1:/{Table/12-Max}] initiating a split of this range at key /Table/13 [r8]
I170517 07:33:19.010955 60482 storage/split_queue.go:92  [split,s1,r8/1:/{Table/13-Max}] splitting at key /Table/14/0
I170517 07:33:19.011133 60482 storage/replica_command.go:2633  [split,s1,r8/1:/{Table/13-Max}] initiating a split of this range at key /Table/14 [r9]
I170517 07:33:19.056204 60482 storage/split_queue.go:92  [split,s1,r9/1:/{Table/14-Max}] splitting at key /Table/50/0
I170517 07:33:19.056398 60482 storage/replica_command.go:2633  [split,s1,r9/1:/{Table/14-Max}] initiating a split of this range at key /Table/50 [r10]
I170517 07:33:19.135884 60482 storage/split_queue.go:92  [split,s1,r10/1:/{Table/50-Max}] splitting at key /Table/51/0
I170517 07:33:19.136068 60482 storage/replica_command.go:2633  [split,s1,r10/1:/{Table/50-Max}] initiating a split of this range at key /Table/51 [r11]
I170517 07:33:19.192622 60482 storage/split_queue.go:92  [split,s1,r11/1:/{Table/51-Max}] splitting at key /Table/52/0
I170517 07:33:19.192843 60482 storage/replica_command.go:2633  [split,s1,r11/1:/{Table/51-Max}] initiating a split of this range at key /Table/52 [r12]
I170517 07:33:19.226406 60482 storage/split_queue.go:92  [split,s1,r12/1:/{Table/52-Max}] splitting at key /Table/53/0
I170517 07:33:19.226846 60482 storage/replica_command.go:2633  [split,s1,r12/1:/{Table/52-Max}] initiating a split of this range at key /Table/53 [r13]
I170517 07:33:19.266167 60482 storage/split_queue.go:92  [split,s1,r13/1:/{Table/53-Max}] splitting at key /Table/54/0
I170517 07:33:19.266370 60482 storage/replica_command.go:2633  [split,s1,r13/1:/{Table/53-Max}] initiating a split of this range at key /Table/54 [r14]
I170517 07:33:19.355706 60482 storage/split_queue.go:92  [split,s1,r14/1:/{Table/54-Max}] splitting at key /Table/55/0
I170517 07:33:19.355875 60482 storage/replica_command.go:2633  [split,s1,r14/1:/{Table/54-Max}] initiating a split of this range at key /Table/55 [r15]
I170517 07:33:19.420606 60482 storage/split_queue.go:92  [split,s1,r15/1:/{Table/55-Max}] splitting at key /Table/56/0
I170517 07:33:19.420762 60482 storage/replica_command.go:2633  [split,s1,r15/1:/{Table/55-Max}] initiating a split of this range at key /Table/56 [r16]
I170517 07:33:19.501909 60482 storage/split_queue.go:92  [split,s1,r16/1:/{Table/56-Max}] splitting at key /Table/57/0
I170517 07:33:19.502057 60482 storage/replica_command.go:2633  [split,s1,r16/1:/{Table/56-Max}] initiating a split of this range at key /Table/57 [r17]
I170517 07:33:19.541766 60482 storage/split_queue.go:92  [split,s1,r17/1:/{Table/57-Max}] splitting at key /Table/58/0
I170517 07:33:19.541938 60482 storage/replica_command.go:2633  [split,s1,r17/1:/{Table/57-Max}] initiating a split of this range at key /Table/58 [r18]
I170517 07:33:19.565360 60482 storage/split_queue.go:92  [split,s1,r18/1:/{Table/58-Max}] splitting at key /Table/59/0
I170517 07:33:19.565619 60482 storage/replica_command.go:2633  [split,s1,r18/1:/{Table/58-Max}] initiating a split of this range at key /Table/59 [r19]
I170517 07:33:19.627686 60482 storage/split_queue.go:92  [split,s1,r19/1:/{Table/59-Max}] splitting at key /Table/60/0
I170517 07:33:19.627858 60482 storage/replica_command.go:2633  [split,s1,r19/1:/{Table/59-Max}] initiating a split of this range at key /Table/60 [r20]
I170517 07:33:19.653168 60482 storage/split_queue.go:92  [split,s1,r20/1:/{Table/60-Max}] splitting at key /Table/61/0
I170517 07:33:19.653510 60482 storage/replica_command.go:2633  [split,s1,r20/1:/{Table/60-Max}] initiating a split of this range at key /Table/61 [r21]
I170517 07:33:19.693923 60482 storage/split_queue.go:92  [split,s1,r21/1:/{Table/61-Max}] splitting at key /Table/62/0
I170517 07:33:19.694108 60482 storage/replica_command.go:2633  [split,s1,r21/1:/{Table/61-Max}] initiating a split of this range at key /Table/62 [r22]
I170517 07:33:19.734053 60482 storage/split_queue.go:92  [split,s1,r22/1:/{Table/62-Max}] splitting at key /Table/63/0
I170517 07:33:19.734273 60482 storage/replica_command.go:2633  [split,s1,r22/1:/{Table/62-Max}] initiating a split of this range at key /Table/63 [r23]
I170517 07:33:19.784903 60482 storage/split_queue.go:92  [split,s1,r23/1:/{Table/63-Max}] splitting at key /Table/64/0
I170517 07:33:19.785085 60482 storage/replica_command.go:2633  [split,s1,r23/1:/{Table/63-Max}] initiating a split of this range at key /Table/64 [r24]
I170517 07:33:19.818114 60482 storage/split_queue.go:92  [split,s1,r24/1:/{Table/64-Max}] splitting at key /Table/65/0
I170517 07:33:19.818290 60482 storage/replica_command.go:2633  [split,s1,r24/1:/{Table/64-Max}] initiating a split of this range at key /Table/65 [r25]
I170517 07:33:19.893755 60482 storage/split_queue.go:92  [split,s1,r25/1:/{Table/65-Max}] splitting at key /Table/66/0
I170517 07:33:19.893928 60482 storage/replica_command.go:2633  [split,s1,r25/1:/{Table/65-Max}] initiating a split of this range at key /Table/66 [r26]
I170517 07:33:19.994730 60482 storage/split_queue.go:92  [split,s1,r26/1:/{Table/66-Max}] splitting at key /Table/67/0
I170517 07:33:19.994910 60482 storage/replica_command.go:2633  [split,s1,r26/1:/{Table/66-Max}] initiating a split of this range at key /Table/67 [r27]
I170517 07:33:20.077288 60482 storage/split_queue.go:92  [split,s1,r27/1:/{Table/67-Max}] splitting at key /Table/68/0
I170517 07:33:20.077476 60482 storage/replica_command.go:2633  [split,s1,r27/1:/{Table/67-Max}] initiating a split of this range at key /Table/68 [r28]
I170517 07:33:20.192795 60482 storage/split_queue.go:92  [split,s1,r28/1:/{Table/68-Max}] splitting at key /Table/69/0
I170517 07:33:20.192960 60482 storage/replica_command.go:2633  [split,s1,r28/1:/{Table/68-Max}] initiating a split of this range at key /Table/69 [r29]
I170517 07:33:20.274270 60482 storage/split_queue.go:92  [split,s1,r29/1:/{Table/69-Max}] splitting at key /Table/70/0
I170517 07:33:20.274430 60482 storage/replica_command.go:2633  [split,s1,r29/1:/{Table/69-Max}] initiating a split of this range at key /Table/70 [r30]
I170517 07:33:20.375472 60482 storage/split_queue.go:92  [split,s1,r30/1:/{Table/70-Max}] splitting at key /Table/71/0
I170517 07:33:20.375656 60482 storage/replica_command.go:2633  [split,s1,r30/1:/{Table/70-Max}] initiating a split of this range at key /Table/71 [r31]
I170517 07:33:20.431632 60403 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/replica_proposal.go:365
1      storage/queue.go:523
I170517 07:33:20.434042 60403 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/queue.go:523
--- PASS: TestStoreRangeSystemSplits (2.21s)
=== RUN   TestSplitSnapshotRace_SplitWins
I170517 07:33:20.597960 60557 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:20.598402 60557 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:57067" > attrs:<> locality:<>
W170517 07:33:20.622886 60557 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:20.631987 60622 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:57067
I170517 07:33:20.658519 60557 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:59168" > attrs:<> locality:<>
W170517 07:33:20.703811 60557 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:33:20.714375 60557 storage/store.go:1252  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I170517 07:33:20.714678 60557 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:43528" > attrs:<> locality:<>
I170517 07:33:20.721001 60914 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:57067
W170517 07:33:20.748157 60557 gossip/gossip.go:1196  [n4] no incoming or outgoing connections
I170517 07:33:20.753799 60740 gossip/client.go:131  [n4] started gossip client to 127.0.0.1:57067
I170517 07:33:20.764625 60557 storage/store.go:1252  [n4,s4]: failed initial metrics computation: [n4,s4]: system config not yet available
I170517 07:33:20.765355 60557 gossip/gossip.go:297  [n4] NodeDescriptor set to node_id:4 address:<network_field:"tcp" address_field:"127.0.0.1:53424" > attrs:<> locality:<>
W170517 07:33:20.782903 60557 gossip/gossip.go:1196  [n5] no incoming or outgoing connections
I170517 07:33:20.785514 60657 gossip/client.go:131  [n5] started gossip client to 127.0.0.1:57067
I170517 07:33:20.789188 60921 gossip/server.go:285  [n1] refusing gossip from node 5 (max 3 conns); forwarding to 4 ({tcp 127.0.0.1:53424})
I170517 07:33:20.802219 60657 gossip/client.go:136  [n5] closing client to node 1 (127.0.0.1:57067): received forward from node 1 to 4 (127.0.0.1:53424)
I170517 07:33:20.804498 61024 gossip/gossip.go:1210  [n5] node has connected to cluster via gossip
I170517 07:33:20.805625 61170 gossip/client.go:131  [n5] started gossip client to 127.0.0.1:53424
I170517 07:33:20.816397 60557 gossip/gossip.go:297  [n5] NodeDescriptor set to node_id:5 address:<network_field:"tcp" address_field:"127.0.0.1:34895" > attrs:<> locality:<>
W170517 07:33:20.878150 60557 gossip/gossip.go:1196  [n6] no incoming or outgoing connections
I170517 07:33:20.881398 61173 gossip/client.go:131  [n6] started gossip client to 127.0.0.1:57067
I170517 07:33:20.882273 61314 gossip/server.go:285  [n1] refusing gossip from node 6 (max 3 conns); forwarding to 4 ({tcp 127.0.0.1:53424})
I170517 07:33:20.895611 61173 gossip/client.go:136  [n6] closing client to node 1 (127.0.0.1:57067): received forward from node 1 to 4 (127.0.0.1:53424)
I170517 07:33:20.896176 61296 gossip/gossip.go:1210  [n6] node has connected to cluster via gossip
I170517 07:33:20.897828 61315 gossip/client.go:131  [n6] started gossip client to 127.0.0.1:53424
I170517 07:33:20.899080 60557 gossip/gossip.go:297  [n6] NodeDescriptor set to node_id:6 address:<network_field:"tcp" address_field:"127.0.0.1:50817" > attrs:<> locality:<>
I170517 07:33:20.928093 60557 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key /System/Max [r2]
I170517 07:33:20.952706 61423 storage/replica_raftstorage.go:416  [s1,r2/1:/{System/Max-Max}] generated preemptive snapshot 1697ac67 at index 11
I170517 07:33:20.955542 61423 storage/store.go:3345  [s1,r2/1:/{System/Max-Max}] streamed snapshot to (n2,s2):?: kv pairs: 29, log entries: 1, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:20.956472 61425 storage/replica_raftstorage.go:597  [s2,r2/?:{-}] applying preemptive snapshot at index 11 (id=1697ac67, encoded size=3585, 1 rocksdb batches, 1 log entries)
I170517 07:33:20.957793 61425 storage/replica_raftstorage.go:605  [s2,r2/?:/{System/Max-Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:20.962434 61423 storage/replica_command.go:3538  [s1,r2/1:/{System/Max-Max}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r2:/{System/Max-Max} [(n1,s1):1, next=2]
W170517 07:33:20.966685 61281 storage/stores.go:218  range not contained in one range: [/Meta2/Max,"\x05\x00"), but have [/Min,/System/Max)
I170517 07:33:20.979335 61476 storage/replica.go:2590  [s1,r2/1:/{System/Max-Max}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:20.984880 61478 storage/replica_raftstorage.go:416  [s1,r2/1:/{System/Max-Max}] generated preemptive snapshot 1692f352 at index 13
I170517 07:33:20.987883 61478 storage/store.go:3345  [s1,r2/1:/{System/Max-Max}] streamed snapshot to (n3,s3):?: kv pairs: 31, log entries: 3, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:20.989105 61460 storage/replica_raftstorage.go:597  [s3,r2/?:{-}] applying preemptive snapshot at index 13 (id=1692f352, encoded size=5100, 1 rocksdb batches, 3 log entries)
I170517 07:33:20.990563 61460 storage/replica_raftstorage.go:605  [s3,r2/?:/{System/Max-Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:20.999271 61478 storage/replica_command.go:3538  [s1,r2/1:/{System/Max-Max}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r2:/{System/Max-Max} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:21.009478 61482 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:21.077674 61510 storage/replica.go:2590  [s1,r2/1:/{System/Max-Max}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:33:21.085417 61512 storage/replica_raftstorage.go:416  [s1,r2/1:/{System/Max-Max}] generated preemptive snapshot 3416596e at index 16
I170517 07:33:21.118479 61514 storage/replica_raftstorage.go:597  [s4,r2/?:{-}] applying preemptive snapshot at index 16 (id=3416596e, encoded size=7015, 1 rocksdb batches, 6 log entries)
I170517 07:33:21.119899 61514 storage/replica_raftstorage.go:605  [s4,r2/?:/{System/Max-Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:21.120404 61512 storage/store.go:3345  [s1,r2/1:/{System/Max-Max}] streamed snapshot to (n4,s4):?: kv pairs: 33, log entries: 6, rate-limit: 2.0 MiB/sec, 5ms
I170517 07:33:21.122543 61512 storage/replica_command.go:3538  [s1,r2/1:/{System/Max-Max}] change replicas (ADD_REPLICA (n4,s4):4): read existing descriptor r2:/{System/Max-Max} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:33:21.166974 61463 storage/replica.go:2590  [s1,r2/1:/{System/Max-Max}] proposing ADD_REPLICA (n4,s4):4: [(n1,s1):1 (n2,s2):2 (n3,s3):3 (n4,s4):4]
I170517 07:33:21.318056 60797 storage/replica_proposal.go:385  [s2,r2/2:/{System/Max-Max}] new range lease repl=(n2,s2):2 start=0.000000123,399 epo=1 pro=0.000000123,400 following repl=(n1,s1):1 start=0.000000000,0 exp=0.900000123,6 pro=0.000000123,7
I170517 07:33:21.354379 61531 storage/replica_command.go:3538  [s2,r2/2:/{System/Max-Max}] change replicas (REMOVE_REPLICA (n1,s1):1): read existing descriptor r2:/{System/Max-Max} [(n1,s1):1, (n2,s2):2, (n3,s3):3, (n4,s4):4, next=5]
I170517 07:33:21.506179 61538 storage/replica.go:2590  [s2,r2/2:/{System/Max-Max}] proposing REMOVE_REPLICA (n1,s1):1: [(n4,s4):4 (n2,s2):2 (n3,s3):3]
I170517 07:33:21.569871 61490 storage/store.go:3152  [s1,r2/1:/{System/Max-Max}] added to replica GC queue (peer suggestion)
I170517 07:33:21.587412 60760 storage/store.go:2139  [replicaGC,s1,r2/1:/{System/Max-Max}] removing replica
I170517 07:33:21.588104 60760 storage/replica.go:684  [replicaGC,s1,r2/1:/{System/Max-Max}] removed 33 (21+12) keys in 0ms [clear=0ms commit=0ms]
I170517 07:33:21.601765 60557 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
I170517 07:33:21.631297 60846 storage/replica_proposal.go:385  [s2,r2/2:/{System/Max-Max}] new range lease repl=(n2,s2):2 start=0.000000123,399 epo=1 pro=1.800000125,5 following repl=(n2,s2):2 start=0.000000123,399 epo=1 pro=0.000000123,400
W170517 07:33:21.639353 61498 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:21.639756 61498 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:21.640331 61523 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):4: store 4 was not found
W170517 07:33:21.640566 61489 storage/raft_transport.go:442  raft transport stream to node 4 failed: store 4 was not found
I170517 07:33:21.646855 61402 storage/replica_command.go:2633  [s2,r2/2:/{System/Max-Max}] initiating a split of this range at key "m" [r11]
I170517 07:33:21.654367 61564 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:21.655361 61586 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:21.655820 61586 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:21.656343 61409 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):4: store 4 was not found
I170517 07:33:21.671562 61568 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:21.672728 61454 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:21.678935 61454 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:21.679224 61603 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):4: store 4 was not found
I170517 07:33:21.703197 61455 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:21.705415 61605 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n4,s4):?
W170517 07:33:21.706198 61605 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:21.706703 61457 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
I170517 07:33:21.712258 60856 storage/replica_proposal.go:385  [s2,r2/2:/{System/Max-Max}] new range lease repl=(n2,s2):2 start=0.000000123,399 epo=1 pro=1.800000125,5 following repl=(n0,s0):? start=0.000000000,0 exp=0.000000000,0
I170517 07:33:21.715215 61618 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:21.726794 61606 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:21.728078 61338 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):4: store 4 was not found
I170517 07:33:21.732132 61573 storage/replica_raftstorage.go:416  [s2,r11/2:{m-/Max}] generated preemptive snapshot cc0b6afb at index 10
I170517 07:33:21.737770 61573 storage/store.go:3345  [s2,r11/2:{m-/Max}] streamed snapshot to (n5,s5):?: kv pairs: 28, log entries: 0, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:21.738974 61575 storage/replica_raftstorage.go:597  [s5,r11/?:{-}] applying preemptive snapshot at index 10 (id=cc0b6afb, encoded size=3570, 1 rocksdb batches, 0 log entries)
I170517 07:33:21.741679 61575 storage/replica_raftstorage.go:605  [s5,r11/?:{m-/Max}] applied preemptive snapshot in 3ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:21.745190 61573 storage/replica_command.go:3538  [s2,r11/2:{m-/Max}] change replicas (ADD_REPLICA (n5,s5):4): read existing descriptor r11:{m-/Max} [(n4,s4):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:33:21.748621 61576 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:21.754678 61607 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):1
W170517 07:33:21.758448 61607 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):1
W170517 07:33:21.758856 61578 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):1: store 4 was not found
I170517 07:33:21.824101 61619 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:21.825618 61640 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n4,s4):?
W170517 07:33:21.827007 61638 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
I170517 07:33:21.831093 61622 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:21.832733 61651 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:21.833497 61609 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):4: store 4 was not found
I170517 07:33:21.836208 61610 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:21.837735 61655 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:21.854284 61653 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):4: store 4 was not found
I170517 07:33:21.860998 61544 storage/replica.go:2590  [s2,r11/2:{m-/Max}] proposing ADD_REPLICA (n5,s5):4: [(n4,s4):1 (n2,s2):2 (n3,s3):3 (n5,s5):4]
I170517 07:33:21.872657 61340 storage/replica_raftstorage.go:416  [s2,r11/2:{m-/Max}] generated preemptive snapshot fc631d8c at index 14
I170517 07:33:21.875343 61340 storage/store.go:3345  [s2,r11/2:{m-/Max}] streamed snapshot to (n6,s6):?: kv pairs: 30, log entries: 4, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:21.878297 61342 storage/replica_raftstorage.go:597  [s6,r11/?:{-}] applying preemptive snapshot at index 14 (id=fc631d8c, encoded size=5916, 1 rocksdb batches, 4 log entries)
I170517 07:33:21.879723 61342 storage/replica_raftstorage.go:605  [s6,r11/?:{m-/Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:21.883330 61340 storage/replica_command.go:3538  [s2,r11/2:{m-/Max}] change replicas (ADD_REPLICA (n6,s6):5): read existing descriptor r11:{m-/Max} [(n4,s4):1, (n2,s2):2, (n3,s3):3, (n5,s5):4, next=5]
I170517 07:33:21.903126 61641 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:21.905455 61645 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n4,s4):?
W170517 07:33:21.906186 61643 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
I170517 07:33:21.936999 61669 storage/replica.go:2590  [s2,r11/2:{m-/Max}] proposing ADD_REPLICA (n6,s6):5: [(n4,s4):1 (n2,s2):2 (n3,s3):3 (n5,s5):4 (n6,s6):5]
I170517 07:33:22.009286 61595 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:22.012403 61582 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n4,s4):?
W170517 07:33:22.013048 61647 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
I170517 07:33:22.102721 61684 storage/raft_transport.go:436  raft transport stream to node 4 established
I170517 07:33:22.111825 61700 storage/replica_command.go:3538  [s2,r11/2:{m-/Max}] change replicas (REMOVE_REPLICA (n3,s3):3): read existing descriptor r11:{m-/Max} [(n4,s4):1, (n2,s2):2, (n3,s3):3, (n5,s5):4, (n6,s6):5, next=6]
W170517 07:33:22.121177 61671 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n4,s4):?
W170517 07:33:22.125158 61686 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
I170517 07:33:22.170273 61612 storage/replica.go:2590  [s2,r11/2:{m-/Max}] proposing REMOVE_REPLICA (n3,s3):3: [(n4,s4):1 (n2,s2):2 (n6,s6):5 (n5,s5):4]
I170517 07:33:22.184265 61490 storage/store.go:3152  [s3,r11/3:{m-/Max}] added to replica GC queue (peer suggestion)
I170517 07:33:22.206756 61613 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:22.208223 61617 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n4,s4):?
W170517 07:33:22.209211 61615 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
I170517 07:33:22.256352 60993 storage/store.go:2139  [replicaGC,s3,r11/3:{m-/Max}] removing replica
I170517 07:33:22.262166 60993 storage/replica.go:684  [replicaGC,s3,r11/3:{m-/Max}] removed 31 (20+11) keys in 5ms [clear=5ms commit=0ms]
I170517 07:33:22.267134 60922 storage/replica_proposal.go:385  [s5,r11/4:{m-/Max}] new range lease repl=(n5,s5):4 start=1.800000125,360 epo=1 pro=1.800000125,361 following repl=(n2,s2):2 start=0.000000123,399 epo=1 pro=1.800000125,5
I170517 07:33:22.278303 61673 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:22.279771 61629 storage/raft_transport.go:257  unable to accept Raft message from (n5,s5):4: no handler registered for (n4,s4):1
W170517 07:33:22.280522 61675 storage/store.go:3156  [s5] raft error: node 4 claims to not contain store 4 for replica (n4,s4):1: store 4 was not found
I170517 07:33:22.288728 61551 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:22.289839 61731 storage/raft_transport.go:257  unable to accept Raft message from (n5,s5):4: no handler registered for (n4,s4):1
W170517 07:33:22.290500 61553 storage/store.go:3156  [s5] raft error: node 4 claims to not contain store 4 for replica (n4,s4):1: store 4 was not found
I170517 07:33:22.303702 61676 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:22.304896 61724 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n4,s4):?
W170517 07:33:22.306086 61678 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
I170517 07:33:22.323214 61721 storage/replica_command.go:3538  [s5,r11/4:{m-/Max}] change replicas (REMOVE_REPLICA (n2,s2):2): read existing descriptor r11:{m-/Max} [(n4,s4):1, (n2,s2):2, (n6,s6):5, (n5,s5):4, next=6]
I170517 07:33:22.354069 61779 storage/replica.go:2590  [s5,r11/4:{m-/Max}] proposing REMOVE_REPLICA (n2,s2):2: [(n4,s4):1 (n5,s5):4 (n6,s6):5]
I170517 07:33:22.374251 61593 storage/store.go:3152  [s2,r11/2:{m-/Max}] added to replica GC queue (peer suggestion)
I170517 07:33:22.375728 61794 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:22.378405 61662 storage/raft_transport.go:257  unable to accept Raft message from (n5,s5):?: no handler registered for (n4,s4):?
W170517 07:33:22.379495 61796 storage/store.go:3156  [s5] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
I170517 07:33:22.418820 61764 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:22.420122 61768 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n4,s4):?
W170517 07:33:22.420816 61766 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
I170517 07:33:22.433170 60810 storage/store.go:2139  [replicaGC,s2,r11/2:{m-/Max}] removing replica
I170517 07:33:22.435925 60557 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/queue.go:523
I170517 07:33:22.436043 60810 storage/replica.go:684  [replicaGC,s2,r11/2:{m-/Max}] removed 32 (20+12) keys in 1ms [clear=0ms commit=1ms]
I170517 07:33:22.477000 61704 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:22.487269 61782 storage/raft_transport.go:257  unable to accept Raft message from (n5,s5):?: no handler registered for (n4,s4):?
W170517 07:33:22.488327 61706 storage/raft_transport.go:476  no handler found for store 5 in response range_id:0 from_replica:<node_id:4 store_id:4 replica_id:0 > to_replica:<node_id:5 store_id:5 replica_id:0 > union:<error:<message:"store 4 was not found" transaction_restart:NONE origin_node:0 detail:<store_not_found:<store_id:4 > > now:<wall_time:0 logical:0 > > >
I170517 07:33:24.031009 61814 storage/replica_proposal.go:385  [s2,r2/2:{/System/Max-m}] new range lease repl=(n2,s2):2 start=0.000000123,399 epo=2 pro=1.800000125,800 following repl=(n2,s2):2 start=0.000000123,399 epo=1 pro=1.800000125,5
I170517 07:33:24.064271 61758 storage/client_test.go:1173  test clock advanced to: 3.600000127,0
I170517 07:33:24.138300 62185 storage/node_liveness.go:250  [hb] heartbeat failed on epoch increment; retrying
I170517 07:33:24.635257 62213 storage/replica_proposal.go:385  [s5,r11/4:{m-/Max}] new range lease repl=(n5,s5):4 start=1.800000125,360 epo=1 pro=3.600000127,53 following repl=(n5,s5):4 start=1.800000125,360 epo=1 pro=1.800000125,361
I170517 07:33:24.657704 62218 storage/replica_proposal.go:385  [s5,r11/4:{m-/Max}] new range lease repl=(n5,s5):4 start=1.800000125,360 epo=2 pro=3.600000127,239 following repl=(n5,s5):4 start=1.800000125,360 epo=1 pro=3.600000127,53
I170517 07:33:24.696328 62074 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/client_test.go:510
I170517 07:33:24.699005 62074 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/client_test.go:510
W170517 07:33:24.700297 62368 storage/replica.go:2296  [hb,s1,r1/1:/{Min-System/Max}] shutdown cancellation after 0.0s of attempting command [txn: d7ab3f99], BeginTransaction [/System/NodeLiveness/3,/Min), ConditionalPut [/System/NodeLiveness/3,/Min), EndTransaction [/System/NodeLiveness/3,/Min)
I170517 07:33:24.702009 61786 storage/node_liveness.go:352  [hb] heartbeat result is ambiguous (server shutdown); retrying
W170517 07:33:24.702794 61786 storage/node_liveness.go:253  [hb] failed node liveness heartbeat: node unavailable; try another peer
W170517 07:33:24.766835 61782 storage/raft_transport.go:257  unable to accept Raft message from (n5,s5):?: no handler registered for (n4,s4):?
W170517 07:33:24.767455 61706 storage/store.go:3156  [s5] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
I170517 07:33:24.806939 60620 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:59168->127.0.0.1:49305: use of closed network connection
I170517 07:33:24.807122 60641 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:33:24.808432 60772 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:59168: operation was canceled"; Reconnecting to {127.0.0.1:59168 <nil>}
I170517 07:33:24.808600 60772 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
I170517 07:33:24.808728 61288 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:50817->127.0.0.1:54682: use of closed network connection
I170517 07:33:24.814346 60643 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:57067->127.0.0.1:45684: use of closed network connection
I170517 07:33:24.815170 60877 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:43528->127.0.0.1:46025: use of closed network connection
--- PASS: TestSplitSnapshotRace_SplitWins (4.26s)
=== RUN   TestSplitSnapshotRace_SnapshotWins
I170517 07:33:24.853657 62377 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:24.853990 62377 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:47128" > attrs:<> locality:<>
W170517 07:33:24.879526 62377 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:24.882323 62543 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:47128
I170517 07:33:24.890754 62377 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:33:24.891042 62377 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:59949" > attrs:<> locality:<>
W170517 07:33:24.918622 62377 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:33:24.922996 62604 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:47128
I170517 07:33:24.939137 62377 storage/store.go:1252  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I170517 07:33:24.939422 62377 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:36862" > attrs:<> locality:<>
W170517 07:33:24.964062 62377 gossip/gossip.go:1196  [n4] no incoming or outgoing connections
I170517 07:33:24.965755 62749 gossip/client.go:131  [n4] started gossip client to 127.0.0.1:47128
I170517 07:33:24.991754 62377 gossip/gossip.go:297  [n4] NodeDescriptor set to node_id:4 address:<network_field:"tcp" address_field:"127.0.0.1:45893" > attrs:<> locality:<>
W170517 07:33:25.046198 62377 gossip/gossip.go:1196  [n5] no incoming or outgoing connections
I170517 07:33:25.048580 62798 gossip/client.go:131  [n5] started gossip client to 127.0.0.1:47128
I170517 07:33:25.049664 62799 gossip/server.go:285  [n1] refusing gossip from node 5 (max 3 conns); forwarding to 4 ({tcp 127.0.0.1:45893})
I170517 07:33:25.053081 62798 gossip/client.go:136  [n5] closing client to node 1 (127.0.0.1:47128): received forward from node 1 to 4 (127.0.0.1:45893)
I170517 07:33:25.054916 62903 gossip/gossip.go:1210  [n5] node has connected to cluster via gossip
I170517 07:33:25.055750 62800 gossip/client.go:131  [n5] started gossip client to 127.0.0.1:45893
I170517 07:33:25.066578 62377 gossip/gossip.go:297  [n5] NodeDescriptor set to node_id:5 address:<network_field:"tcp" address_field:"127.0.0.1:43347" > attrs:<> locality:<>
W170517 07:33:25.094963 62377 gossip/gossip.go:1196  [n6] no incoming or outgoing connections
I170517 07:33:25.105580 62377 storage/store.go:1252  [n6,s6]: failed initial metrics computation: [n6,s6]: system config not yet available
I170517 07:33:25.105800 62377 gossip/gossip.go:297  [n6] NodeDescriptor set to node_id:6 address:<network_field:"tcp" address_field:"127.0.0.1:49041" > attrs:<> locality:<>
I170517 07:33:25.108323 63154 gossip/client.go:131  [n6] started gossip client to 127.0.0.1:47128
I170517 07:33:25.109414 63046 gossip/server.go:285  [n1] refusing gossip from node 6 (max 3 conns); forwarding to 2 ({tcp 127.0.0.1:59949})
I170517 07:33:25.118272 63154 gossip/client.go:136  [n6] closing client to node 1 (127.0.0.1:47128): received forward from node 1 to 2 (127.0.0.1:59949)
I170517 07:33:25.118999 63064 gossip/gossip.go:1210  [n6] node has connected to cluster via gossip
I170517 07:33:25.121880 62993 gossip/client.go:131  [n6] started gossip client to 127.0.0.1:59949
I170517 07:33:25.153357 62377 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key /System/Max [r2]
I170517 07:33:25.172905 63223 storage/replica_raftstorage.go:416  [s1,r2/1:/{System/Max-Max}] generated preemptive snapshot 2df960ca at index 11
I170517 07:33:25.175875 63223 storage/store.go:3345  [s1,r2/1:/{System/Max-Max}] streamed snapshot to (n2,s2):?: kv pairs: 29, log entries: 1, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:25.176768 63193 storage/replica_raftstorage.go:597  [s2,r2/?:{-}] applying preemptive snapshot at index 11 (id=2df960ca, encoded size=3585, 1 rocksdb batches, 1 log entries)
I170517 07:33:25.178370 63193 storage/replica_raftstorage.go:605  [s2,r2/?:/{System/Max-Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:25.180861 63223 storage/replica_command.go:3538  [s1,r2/1:/{System/Max-Max}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r2:/{System/Max-Max} [(n1,s1):1, next=2]
W170517 07:33:25.187223 63237 storage/stores.go:218  range not contained in one range: [/Meta2/Max,"\x05\x00"), but have [/Min,/System/Max)
I170517 07:33:25.193814 63241 storage/replica.go:2590  [s1,r2/1:/{System/Max-Max}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:25.213386 63243 storage/replica_raftstorage.go:416  [s1,r2/1:/{System/Max-Max}] generated preemptive snapshot b36637b3 at index 13
I170517 07:33:25.215917 63243 storage/store.go:3345  [s1,r2/1:/{System/Max-Max}] streamed snapshot to (n3,s3):?: kv pairs: 31, log entries: 3, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:25.218279 63226 storage/replica_raftstorage.go:597  [s3,r2/?:{-}] applying preemptive snapshot at index 13 (id=b36637b3, encoded size=5100, 1 rocksdb batches, 3 log entries)
I170517 07:33:25.223327 63226 storage/replica_raftstorage.go:605  [s3,r2/?:/{System/Max-Max}] applied preemptive snapshot in 5ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:33:25.225980 63243 storage/replica_command.go:3538  [s1,r2/1:/{System/Max-Max}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r2:/{System/Max-Max} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:25.240068 63245 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:25.260272 63196 storage/replica.go:2590  [s1,r2/1:/{System/Max-Max}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:33:25.271060 63252 storage/replica_raftstorage.go:416  [s1,r2/1:/{System/Max-Max}] generated preemptive snapshot 1be926c2 at index 16
I170517 07:33:25.275335 63268 storage/replica_raftstorage.go:597  [s4,r2/?:{-}] applying preemptive snapshot at index 16 (id=1be926c2, encoded size=7015, 1 rocksdb batches, 6 log entries)
I170517 07:33:25.276868 63268 storage/replica_raftstorage.go:605  [s4,r2/?:/{System/Max-Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:33:25.277506 63252 storage/store.go:3345  [s1,r2/1:/{System/Max-Max}] streamed snapshot to (n4,s4):?: kv pairs: 33, log entries: 6, rate-limit: 2.0 MiB/sec, 6ms
I170517 07:33:25.279481 63252 storage/replica_command.go:3538  [s1,r2/1:/{System/Max-Max}] change replicas (ADD_REPLICA (n4,s4):4): read existing descriptor r2:/{System/Max-Max} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:33:25.344235 63270 storage/replica.go:2590  [s1,r2/1:/{System/Max-Max}] proposing ADD_REPLICA (n4,s4):4: [(n1,s1):1 (n2,s2):2 (n3,s3):3 (n4,s4):4]
I170517 07:33:25.523222 62585 storage/replica_proposal.go:385  [s2,r2/2:/{System/Max-Max}] new range lease repl=(n2,s2):2 start=0.000000123,395 epo=1 pro=0.000000123,396 following repl=(n1,s1):1 start=0.000000000,0 exp=0.900000123,6 pro=0.000000123,7
I170517 07:33:25.559624 63299 storage/replica_command.go:3538  [s2,r2/2:/{System/Max-Max}] change replicas (REMOVE_REPLICA (n1,s1):1): read existing descriptor r2:/{System/Max-Max} [(n1,s1):1, (n2,s2):2, (n3,s3):3, (n4,s4):4, next=5]
I170517 07:33:25.642109 63279 storage/replica.go:2590  [s2,r2/2:/{System/Max-Max}] proposing REMOVE_REPLICA (n1,s1):1: [(n4,s4):4 (n2,s2):2 (n3,s3):3]
I170517 07:33:25.668567 63231 storage/store.go:3152  [s1,r2/1:/{System/Max-Max}] added to replica GC queue (peer suggestion)
I170517 07:33:25.689100 62532 storage/store.go:2139  [replicaGC,s1,r2/1:/{System/Max-Max}] removing replica
I170517 07:33:25.689902 62532 storage/replica.go:684  [replicaGC,s1,r2/1:/{System/Max-Max}] removed 33 (21+12) keys in 0ms [clear=0ms commit=0ms]
W170517 07:33:25.704397 63172 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n4,s4):?
I170517 07:33:25.704643 62377 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
W170517 07:33:25.705215 63141 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
W170517 07:33:25.705495 63139 storage/raft_transport.go:442  raft transport stream to node 4 failed: store 4 was not found
I170517 07:33:25.736452 63321 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:25.740260 63307 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:25.741049 63323 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):4: store 4 was not found
I170517 07:33:25.743624 63265 storage/raft_transport.go:436  raft transport stream to node 4 established
I170517 07:33:25.744037 62589 storage/replica_proposal.go:385  [s2,r2/2:/{System/Max-Max}] new range lease repl=(n2,s2):2 start=0.000000123,395 epo=1 pro=1.800000125,5 following repl=(n2,s2):2 start=0.000000123,395 epo=1 pro=0.000000123,396
W170517 07:33:25.744620 63295 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:25.745354 63177 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):4: store 4 was not found
I170517 07:33:25.763827 63316 storage/replica_command.go:2633  [s2,r2/2:/{System/Max-Max}] initiating a split of this range at key "m" [r11]
I170517 07:33:25.767498 63346 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:25.768451 63297 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:25.769129 63348 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):4: store 4 was not found
I170517 07:33:25.771447 63362 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:25.772778 63350 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:25.778424 63364 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):4: store 4 was not found
W170517 07:33:25.779296 63350 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
I170517 07:33:25.802341 62565 storage/replica_proposal.go:385  [s2,r2/2:/{System/Max-Max}] new range lease repl=(n2,s2):2 start=0.000000123,395 epo=1 pro=1.800000125,5 following repl=(n0,s0):? start=0.000000000,0 exp=0.000000000,0
I170517 07:33:25.806788 63145 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:25.807708 63310 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:25.808096 63310 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:25.808265 63310 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n4,s4):?
W170517 07:33:25.808379 63380 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):4: store 4 was not found
W170517 07:33:25.808480 63310 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
I170517 07:33:25.811442 63148 storage/replica_raftstorage.go:416  [s2,r11/2:{m-/Max}] generated preemptive snapshot 12370a20 at index 10
I170517 07:33:25.814727 63148 storage/store.go:3345  [s2,r11/2:{m-/Max}] streamed snapshot to (n5,s5):?: kv pairs: 28, log entries: 0, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:25.815343 63179 storage/replica_raftstorage.go:597  [s5,r11/?:{-}] applying preemptive snapshot at index 10 (id=12370a20, encoded size=3570, 1 rocksdb batches, 0 log entries)
I170517 07:33:25.816442 63179 storage/replica_raftstorage.go:605  [s5,r11/?:{m-/Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:25.819912 63148 storage/replica_command.go:3538  [s2,r11/2:{m-/Max}] change replicas (ADD_REPLICA (n5,s5):4): read existing descriptor r11:{m-/Max} [(n4,s4):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:33:25.823015 63383 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:25.824479 63387 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):1
W170517 07:33:25.825144 63385 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):1: store 4 was not found
I170517 07:33:25.850197 63356 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:25.853279 63394 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:25.853656 63394 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n4,s4):4
W170517 07:33:25.857605 63358 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):4: store 4 was not found
I170517 07:33:25.859284 63149 storage/replica.go:2590  [s2,r11/2:{m-/Max}] proposing ADD_REPLICA (n5,s5):4: [(n4,s4):1 (n2,s2):2 (n3,s3):3 (n5,s5):4]
I170517 07:33:25.899330 63181 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:25.900329 63371 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n4,s4):?
W170517 07:33:25.901049 63396 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
I170517 07:33:25.905995 63373 storage/replica_raftstorage.go:416  [s2,r11/2:{m-/Max}] generated preemptive snapshot 6ed41f39 at index 14
I170517 07:33:25.908903 63373 storage/store.go:3345  [s2,r11/2:{m-/Max}] streamed snapshot to (n6,s6):?: kv pairs: 30, log entries: 4, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:25.910217 63376 storage/replica_raftstorage.go:597  [s6,r11/?:{-}] applying preemptive snapshot at index 14 (id=6ed41f39, encoded size=5916, 1 rocksdb batches, 4 log entries)
I170517 07:33:25.911585 63376 storage/replica_raftstorage.go:605  [s6,r11/?:{m-/Max}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:25.914214 63373 storage/replica_command.go:3538  [s2,r11/2:{m-/Max}] change replicas (ADD_REPLICA (n6,s6):5): read existing descriptor r11:{m-/Max} [(n4,s4):1, (n2,s2):2, (n3,s3):3, (n5,s5):4, next=5]
I170517 07:33:25.945612 63150 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:25.948170 63442 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n4,s4):?
W170517 07:33:25.949416 63152 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
I170517 07:33:25.974113 63332 storage/replica.go:2590  [s2,r11/2:{m-/Max}] proposing ADD_REPLICA (n6,s6):5: [(n4,s4):1 (n2,s2):2 (n3,s3):3 (n5,s5):4 (n6,s6):5]
I170517 07:33:25.995721 63428 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:26.003518 63434 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n4,s4):?
W170517 07:33:26.004324 63431 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
I170517 07:33:26.025806 63401 storage/replica_command.go:3538  [s2,r11/2:{m-/Max}] change replicas (REMOVE_REPLICA (n3,s3):3): read existing descriptor r11:{m-/Max} [(n4,s4):1, (n2,s2):2, (n3,s3):3, (n5,s5):4, (n6,s6):5, next=6]
I170517 07:33:26.074360 63446 storage/replica.go:2590  [s2,r11/2:{m-/Max}] proposing REMOVE_REPLICA (n3,s3):3: [(n4,s4):1 (n2,s2):2 (n6,s6):5 (n5,s5):4]
I170517 07:33:26.125994 63231 storage/store.go:3152  [s3,r11/3:{m-/Max}] added to replica GC queue (peer suggestion)
I170517 07:33:26.144130 62424 storage/store.go:2139  [replicaGC,s3,r11/3:{m-/Max}] removing replica
I170517 07:33:26.145532 63447 storage/raft_transport.go:436  raft transport stream to node 4 established
I170517 07:33:26.145978 62424 storage/replica.go:684  [replicaGC,s3,r11/3:{m-/Max}] removed 31 (20+11) keys in 0ms [clear=0ms commit=0ms]
W170517 07:33:26.149064 63461 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n4,s4):?
W170517 07:33:26.149793 63449 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
I170517 07:33:26.151114 62924 storage/replica_proposal.go:385  [s5,r11/4:{m-/Max}] new range lease repl=(n5,s5):4 start=1.800000125,317 epo=1 pro=1.800000125,318 following repl=(n2,s2):2 start=0.000000123,395 epo=1 pro=1.800000125,5
I170517 07:33:26.154102 63393 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:26.171914 63408 storage/raft_transport.go:257  unable to accept Raft message from (n5,s5):4: no handler registered for (n4,s4):1
W170517 07:33:26.172373 63408 storage/raft_transport.go:257  unable to accept Raft message from (n5,s5):4: no handler registered for (n4,s4):1
W170517 07:33:26.181419 63475 storage/store.go:3156  [s5] raft error: node 4 claims to not contain store 4 for replica (n4,s4):1: store 4 was not found
I170517 07:33:26.184209 63438 storage/replica_command.go:3538  [s5,r11/4:{m-/Max}] change replicas (REMOVE_REPLICA (n2,s2):2): read existing descriptor r11:{m-/Max} [(n4,s4):1, (n2,s2):2, (n6,s6):5, (n5,s5):4, next=6]
I170517 07:33:26.206744 63414 storage/replica.go:2590  [s5,r11/4:{m-/Max}] proposing REMOVE_REPLICA (n2,s2):2: [(n4,s4):1 (n5,s5):4 (n6,s6):5]
I170517 07:33:26.241345 63410 storage/store.go:3152  [s2,r11/2:{m-/Max}] added to replica GC queue (peer suggestion)
I170517 07:33:26.245398 63469 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:26.246201 63482 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n4,s4):?
W170517 07:33:26.246826 63440 storage/store.go:3156  [s2] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
I170517 07:33:26.262446 62598 storage/store.go:2139  [replicaGC,s2,r11/2:{m-/Max}] removing replica
I170517 07:33:26.263246 62598 storage/replica.go:684  [replicaGC,s2,r11/2:{m-/Max}] removed 32 (20+12) keys in 0ms [clear=0ms commit=0ms]
I170517 07:33:26.265629 63338 storage/raft_transport.go:436  raft transport stream to node 4 established
W170517 07:33:26.266772 63484 storage/raft_transport.go:257  unable to accept Raft message from (n5,s5):?: no handler registered for (n4,s4):?
W170517 07:33:26.278230 63340 storage/store.go:3156  [s5] raft error: node 4 claims to not contain store 4 for replica (n4,s4):?: store 4 was not found
W170517 07:33:26.282752 63162 storage/raft_transport.go:257  unable to accept Raft message from (n3,s3):?: no handler registered for (n2,s2):?
W170517 07:33:26.283661 63231 storage/store.go:3156  [s3] raft error: node 2 claims to not contain store 2 for replica (n2,s2):?: store 2 was not found
W170517 07:33:26.283885 63229 storage/raft_transport.go:442  raft transport stream to node 2 failed: store 2 was not found
I170517 07:33:26.815476 63489 storage/raft_transport.go:436  raft transport stream to node 4 established
I170517 07:33:26.825919 63684 storage/replica_raftstorage.go:416  [raftsnapshot,s5,r11/4:{m-/Max}] generated Raft snapshot 97641068 at index 28
E170517 07:33:26.833662 63684 storage/queue.go:634  [raftsnapshot,s5,r11/4:{m-/Max}] snapshot failed: (n4,s4):1: remote couldn't accept snapshot with error: [n4,s4],r11: cannot apply snapshot: snapshot intersects existing range; initiated GC: [n4,s4,r2/4:/{System/Max-Max}]
I170517 07:33:26.838837 63616 storage/replica_proposal.go:385  [s5,r11/4:{m-/Max}] new range lease repl=(n5,s5):4 start=1.800000125,317 epo=1 pro=1.800000125,656 following repl=(n5,s5):4 start=1.800000125,317 epo=1 pro=1.800000125,318
I170517 07:33:26.840547 63835 storage/client_test.go:1173  [replicaGC,r2/4:/{System/Max-Max},s1] test clock advanced to: 3.600000127,0
I170517 07:33:26.901568 63620 storage/replica_proposal.go:385  [s5,r11/4:{m-/Max}] new range lease repl=(n5,s5):4 start=1.800000125,317 epo=2 pro=3.600000127,2 following repl=(n5,s5):4 start=1.800000125,317 epo=1 pro=1.800000125,656
I170517 07:33:26.999619 63684 storage/replica_raftstorage.go:416  [raftsnapshot,s5,r11/4:{m-/Max}] generated Raft snapshot 27d911f0 at index 31
E170517 07:33:27.005507 63684 storage/queue.go:634  [raftsnapshot,s5,r11/4:{m-/Max}] snapshot failed: (n4,s4):1: remote couldn't accept snapshot with error: [n4,s4],r11: cannot apply snapshot: snapshot intersects existing range; initiated GC: [n4,s4,r2/4:/{System/Max-Max}]
I170517 07:33:27.073676 64084 storage/raft_transport.go:436  raft transport stream to node 2 established
I170517 07:33:27.084768 63684 storage/replica_raftstorage.go:416  [raftsnapshot,s5,r11/4:{m-/Max}] generated Raft snapshot 2fde63e7 at index 31
E170517 07:33:27.089698 63684 storage/queue.go:634  [raftsnapshot,s5,r11/4:{m-/Max}] snapshot failed: (n4,s4):1: remote couldn't accept snapshot with error: [n4,s4],r11: cannot apply snapshot: snapshot intersects existing range; initiated GC: [n4,s4,r2/4:/{System/Max-Max}]
I170517 07:33:27.125355 63684 storage/replica_raftstorage.go:416  [raftsnapshot,s5,r11/4:{m-/Max}] generated Raft snapshot 678ba5b7 at index 31
E170517 07:33:27.128084 63684 storage/queue.go:634  [raftsnapshot,s5,r11/4:{m-/Max}] snapshot failed: (n4,s4):1: remote couldn't accept snapshot with error: [n4,s4],r11: cannot apply snapshot: snapshot intersects existing range [n4,s4,r2/4:/{System/Max-Max}]
I170517 07:33:27.274418 63684 storage/replica_raftstorage.go:416  [raftsnapshot,s5,r11/4:{m-/Max}] generated Raft snapshot 4080eed5 at index 31
E170517 07:33:27.277437 63684 storage/queue.go:634  [raftsnapshot,s5,r11/4:{m-/Max}] snapshot failed: (n4,s4):1: remote couldn't accept snapshot with error: [n4,s4],r11: cannot apply snapshot: snapshot intersects existing range [n4,s4,r2/4:/{System/Max-Max}]
I170517 07:33:27.323999 63684 storage/replica_raftstorage.go:416  [raftsnapshot,s5,r11/4:{m-/Max}] generated Raft snapshot 9ada847f at index 31
E170517 07:33:27.326740 63684 storage/queue.go:634  [raftsnapshot,s5,r11/4:{m-/Max}] snapshot failed: (n4,s4):1: remote couldn't accept snapshot with error: [n4,s4],r11: cannot apply snapshot: snapshot intersects existing range [n4,s4,r2/4:/{System/Max-Max}]
I170517 07:33:27.478992 63684 storage/replica_raftstorage.go:416  [raftsnapshot,s5,r11/4:{m-/Max}] generated Raft snapshot c693d07d at index 31
E170517 07:33:27.481597 63684 storage/queue.go:634  [raftsnapshot,s5,r11/4:{m-/Max}] snapshot failed: (n4,s4):1: remote couldn't accept snapshot with error: [n4,s4],r11: cannot apply snapshot: snapshot intersects existing range [n4,s4,r2/4:/{System/Max-Max}]
I170517 07:33:27.576803 63684 storage/replica_raftstorage.go:416  [raftsnapshot,s5,r11/4:{m-/Max}] generated Raft snapshot 5d07bc45 at index 31
E170517 07:33:27.579485 63684 storage/queue.go:634  [raftsnapshot,s5,r11/4:{m-/Max}] snapshot failed: (n4,s4):1: remote couldn't accept snapshot with error: [n4,s4],r11: cannot apply snapshot: snapshot intersects existing range [n4,s4,r2/4:/{System/Max-Max}]
I170517 07:33:27.625644 63684 storage/replica_raftstorage.go:416  [raftsnapshot,s5,r11/4:{m-/Max}] generated Raft snapshot 84717951 at index 31
E170517 07:33:27.628702 63684 storage/queue.go:634  [raftsnapshot,s5,r11/4:{m-/Max}] snapshot failed: (n4,s4):1: remote couldn't accept snapshot with error: [n4,s4],r11: cannot apply snapshot: snapshot intersects existing range [n4,s4,r2/4:/{System/Max-Max}]
I170517 07:33:27.664212 63865 storage/replica_proposal.go:385  [s2,r2/2:{/System/Max-m}] new range lease repl=(n2,s2):2 start=0.000000123,395 epo=3 pro=3.600000127,199 following repl=(n2,s2):2 start=0.000000123,395 epo=1 pro=1.800000125,5
I170517 07:33:27.703616 64076 storage/client_test.go:1173  test clock advanced to: 5.400000129,0
I170517 07:33:27.727108 64071 storage/client_test.go:1173  [replicaGC,r2/4:{/System/Max-m},s1] test clock advanced to: 7.200000131,0
I170517 07:33:27.772346 63651 storage/replica_proposal.go:385  [s5,r11/4:{m-/Max}] new range lease repl=(n5,s5):4 start=1.800000125,317 epo=2 pro=7.200000131,28 following repl=(n5,s5):4 start=1.800000125,317 epo=2 pro=3.600000127,2
I170517 07:33:27.776333 63818 storage/client_test.go:1173  [replicaGC,r2/4:{/System/Max-m},s1] test clock advanced to: 9.000000133,0
I170517 07:33:27.783738 63684 storage/replica_raftstorage.go:416  [raftsnapshot,s5,r11/4:{m-/Max}] generated Raft snapshot 26c23b08 at index 32
I170517 07:33:27.801228 63684 storage/store.go:3345  [raftsnapshot,s5,r11/4:{m-/Max}] streamed snapshot to (n4,s4):1: kv pairs: 38, log entries: 22, rate-limit: 8.0 MiB/sec, 17ms
I170517 07:33:27.805419 64179 storage/replica_raftstorage.go:597  [s4,r11/1:{m-/Max}] applying Raft snapshot at index 32 (id=26c23b08, encoded size=15225, 1 rocksdb batches, 22 log entries)
I170517 07:33:27.811819 64179 storage/replica_raftstorage.go:605  [s4,r11/1:{m-/Max}] applied Raft snapshot in 6ms [clear=0ms batch=0ms entries=4ms commit=0ms]
I170517 07:33:27.847141 63657 storage/replica_proposal.go:385  [s5,r11/4:{m-/Max}] new range lease repl=(n5,s5):4 start=1.800000125,317 epo=2 pro=9.000000133,10 following repl=(n5,s5):4 start=1.800000125,317 epo=2 pro=7.200000131,28
I170517 07:33:27.856711 63890 storage/replica_proposal.go:385  [s2,r2/2:{/System/Max-m}] new range lease repl=(n2,s2):2 start=0.000000123,395 epo=3 pro=9.000000133,60 following repl=(n2,s2):2 start=0.000000123,395 epo=3 pro=3.600000127,199
W170517 07:33:27.909028 64168 storage/replica.go:2296  [replicaGC,s1,r1/1:/{Min-System/Max}] shutdown cancellation after 0.0s of attempting command ResolveIntent [/Meta2/"m",/Min)
I170517 07:33:27.912601 64204 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/queue.go:523
I170517 07:33:27.914690 64201 util/stop/stopper.go:505  quiescing; tasks left:
4      storage/client_test.go:510
I170517 07:33:27.914925 64201 util/stop/stopper.go:505  quiescing; tasks left:
3      storage/client_test.go:510
I170517 07:33:27.915112 64201 util/stop/stopper.go:505  quiescing; tasks left:
2      storage/client_test.go:510
I170517 07:33:27.915292 64201 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/client_test.go:510
W170517 07:33:27.927444 64169 storage/replica.go:2296  [hb,s1,r1/1:/{Min-System/Max}] shutdown cancellation after 0.0s of attempting command [txn: d6886d38], BeginTransaction [/System/NodeLiveness/6,/Min), ConditionalPut [/System/NodeLiveness/6,/Min), EndTransaction [/System/NodeLiveness/6,/Min)
I170517 07:33:27.928751 63780 storage/node_liveness.go:352  [hb] heartbeat result is ambiguous (server shutdown); retrying
W170517 07:33:27.929404 63780 storage/node_liveness.go:253  [hb] failed node liveness heartbeat: node unavailable; try another peer
I170517 07:33:28.022725 62680 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:36862->127.0.0.1:59888: use of closed network connection
I170517 07:33:28.029195 62791 gossip/gossip.go:1210  [n4] node has connected to cluster via gossip
I170517 07:33:28.032214 62682 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:33:28.034254 62283 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:33:28.035678 62685 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:36862: getsockopt: connection refused"; Reconnecting to {127.0.0.1:36862 <nil>}
I170517 07:33:28.036467 62286 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:47128: getsockopt: connection refused"; Reconnecting to {127.0.0.1:47128 <nil>}
I170517 07:33:28.038044 63059 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:49041->127.0.0.1:34671: use of closed network connection
I170517 07:33:28.038888 62434 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:47128->127.0.0.1:40290: use of closed network connection
I170517 07:33:28.039078 62685 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: context canceled
I170517 07:33:28.039548 63038 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:33:28.040961 63041 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:49041: getsockopt: connection reset by peer"; Reconnecting to {127.0.0.1:49041 <nil>}
I170517 07:33:28.041107 63041 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
I170517 07:33:28.042652 62286 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: context canceled
I170517 07:33:28.043337 62540 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:59949->127.0.0.1:50735: use of closed network connection
--- PASS: TestSplitSnapshotRace_SnapshotWins (3.26s)
=== RUN   TestStoreSplitTimestampCacheDifferentLeaseHolder
W170517 07:33:28.252831 64182 server/status/runtime.go:111  Could not parse build timestamp: parsing time "" as "2006/01/02 15:04:05": cannot parse "" as "2006"
I170517 07:33:28.256655 64182 server/config.go:436  1 storage engine initialized
I170517 07:33:28.260617 64182 server/node.go:454  [n?] store [n0,s0] not bootstrapped
I170517 07:33:28.292884 64182 server/node.go:389  [n?] **** cluster acba34b3-8ed6-42b4-8b21-f4928ea567cf has been created
I170517 07:33:28.292983 64182 server/node.go:390  [n?] **** add additional nodes by specifying --join=127.0.0.1:35659
I170517 07:33:28.302460 64182 storage/store.go:1252  [n1] [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:28.308274 64182 server/node.go:467  [n1] initialized store [n1,s1]: {Capacity:536870912 Available:536870912 RangeCount:1 LeaseCount:0}
I170517 07:33:28.308493 64182 server/node.go:351  [n1] node ID 1 initialized
I170517 07:33:28.308711 64182 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:35659" > attrs:<> locality:<>
I170517 07:33:28.309659 64182 storage/stores.go:296  [n1] read 0 node addresses from persistent storage
I170517 07:33:28.309992 64182 server/node.go:608  [n1] connecting to gossip network to verify cluster ID...
I170517 07:33:28.310116 64182 server/node.go:633  [n1] node connected via gossip and verified as part of cluster "acba34b3-8ed6-42b4-8b21-f4928ea567cf"
I170517 07:33:28.312731 64182 server/node.go:405  [n1] node=1: started with [[]=] engine(s) and attributes []
I170517 07:33:28.320484 64182 sql/executor.go:349  [n1] creating distSQLPlanner with address {tcp 127.0.0.1:35659}
I170517 07:33:28.351588 64182 server/server.go:736  [n1] starting https server at 127.0.0.1:39454
I170517 07:33:28.351775 64182 server/server.go:737  [n1] starting grpc/postgres server at 127.0.0.1:35659
I170517 07:33:28.351847 64182 server/server.go:738  [n1] advertising CockroachDB node at 127.0.0.1:35659
I170517 07:33:28.442747 64182 sql/event_log.go:101  [n1] Event: "alter_table", target: 12, info: {TableName:eventlog Statement:ALTER TABLE system.eventlog ALTER COLUMN uniqueid SET DEFAULT uuid_v4() User:node MutationID:0 CascadeDroppedViews:[]}
I170517 07:33:28.571984 64182 sql/lease.go:372  [n1] publish: descID=12 (eventlog) version=2 mtime=2017-05-17 07:33:28.571893651 +0000 UTC
I170517 07:33:28.750316 64182 server/server.go:848  [n1] done ensuring all necessary migrations have run
I170517 07:33:28.750439 64182 server/server.go:850  [n1] serving sql connections
I170517 07:33:28.811243 64503 sql/event_log.go:101  [n1] Event: "node_join", target: 1, info: {Descriptor:{NodeID:1 Address:{NetworkField:tcp AddressField:127.0.0.1:35659} Attrs: Locality:} ClusterID:acba34b3-8ed6-42b4-8b21-f4928ea567cf StartedAt:1495006408310149121 LastUp:1495006408310149121}
W170517 07:33:28.852197 64182 server/status/runtime.go:111  Could not parse build timestamp: parsing time "" as "2006/01/02 15:04:05": cannot parse "" as "2006"
W170517 07:33:28.877633 64182 gossip/gossip.go:1196  [n?] no incoming or outgoing connections
I170517 07:33:28.879262 64182 server/config.go:436  1 storage engine initialized
I170517 07:33:28.901366 64182 server/node.go:454  [n?] store [n0,s0] not bootstrapped
I170517 07:33:28.901558 64182 storage/stores.go:296  [n?] read 0 node addresses from persistent storage
I170517 07:33:28.901663 64182 server/node.go:608  [n?] connecting to gossip network to verify cluster ID...
I170517 07:33:28.992371 64456 gossip/client.go:131  [n?] started gossip client to 127.0.0.1:35659
I170517 07:33:28.994577 64464 gossip/server.go:234  [n1] received initial cluster-verification connection from {tcp 127.0.0.1:52031}
I170517 07:33:29.013283 64182 server/node.go:633  [n?] node connected via gossip and verified as part of cluster "acba34b3-8ed6-42b4-8b21-f4928ea567cf"
I170517 07:33:29.014808 64432 storage/stores.go:312  [n?] wrote 1 node addresses to persistent storage
I170517 07:33:29.021905 64182 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
I170517 07:33:29.029498 64182 server/node.go:344  [n?] new node allocated ID 2
I170517 07:33:29.029789 64182 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:52031" > attrs:<> locality:<>
I170517 07:33:29.030721 64182 server/node.go:405  [n2] node=2: started with [[]=] engine(s) and attributes []
I170517 07:33:29.033075 64182 sql/executor.go:349  [n2] creating distSQLPlanner with address {tcp 127.0.0.1:52031}
I170517 07:33:29.046056 64626 storage/stores.go:312  [n1] wrote 1 node addresses to persistent storage
I170517 07:33:29.128390 64571 server/node.go:589  [n2] bootstrapped store [n2,s2]
I170517 07:33:29.158858 64182 server/server.go:736  [n2] starting https server at 127.0.0.1:45395
I170517 07:33:29.159093 64182 server/server.go:737  [n2] starting grpc/postgres server at 127.0.0.1:52031
I170517 07:33:29.159270 64182 server/server.go:738  [n2] advertising CockroachDB node at 127.0.0.1:52031
I170517 07:33:29.229663 64182 server/server.go:848  [n2] done ensuring all necessary migrations have run
I170517 07:33:29.229803 64182 server/server.go:850  [n2] serving sql connections
I170517 07:33:29.265895 64719 storage/replica_command.go:2633  [n1,s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r2]
I170517 07:33:29.295990 64715 sql/event_log.go:101  [n2] Event: "node_join", target: 2, info: {Descriptor:{NodeID:2 Address:{NetworkField:tcp AddressField:127.0.0.1:52031} Attrs: Locality:} ClusterID:acba34b3-8ed6-42b4-8b21-f4928ea567cf StartedAt:1495006409030440602 LastUp:1495006409030440602}
I170517 07:33:29.350905 64625 storage/replica_command.go:2633  [n1,s1,r2/1:{a-/Max}] initiating a split of this range at key "c" [r3]
I170517 07:33:29.484399 64810 storage/replica_raftstorage.go:416  [n1,s1,r2/1:{a-c}] generated preemptive snapshot 60c40581 at index 16
I170517 07:33:29.601008 64810 storage/store.go:3345  [n1,s1,r2/1:{a-c}] streamed snapshot to (n2,s2):?: kv pairs: 10, log entries: 6, rate-limit: 2.0 MiB/sec, 5ms
I170517 07:33:29.606136 64814 storage/replica_raftstorage.go:597  [n2,s2,r2/?:{-}] applying preemptive snapshot at index 16 (id=60c40581, encoded size=7480, 1 rocksdb batches, 6 log entries)
I170517 07:33:29.607825 64814 storage/replica_raftstorage.go:605  [n2,s2,r2/?:{a-c}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:33:29.611247 64810 storage/replica_command.go:3538  [n1,s1,r2/1:{a-c}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r2:{a-c} [(n1,s1):1, next=2]
I170517 07:33:29.635201 64837 storage/replica.go:2590  [n1,s1,r2/1:{a-c}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:29.660648 64867 storage/raft_transport.go:436  [n2] raft transport stream to node 1 established
I170517 07:33:29.671288 64182 storage/client_split_test.go:1221  blacklisting replica (n1,s1):1 for leases
I170517 07:33:29.671486 64182 storage/client_split_test.go:1236  splitting at "b"
I170517 07:33:29.672185 64873 storage/replica_command.go:2633  [n1,s1,r2/1:{a-c}] initiating a split of this range at key "b" [r4]
I170517 07:33:29.862596 64857 util/stop/stopper.go:505  quiescing; tasks left:
1      kv/txn_coord_sender.go:981
I170517 07:33:29.865797 64060 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:35659->127.0.0.1:55648: use of closed network connection
I170517 07:33:29.870734 64191 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:35659->127.0.0.1:55656: use of closed network connection
W170517 07:33:29.873511 64867 storage/raft_transport.go:442  [n2] raft transport stream to node 1 failed: rpc error: code = Internal desc = transport is closing
I170517 07:33:29.899608 64445 kv/transport_race.go:71  transport race promotion: ran 23 iterations on up to 179 requests
--- PASS: TestStoreSplitTimestampCacheDifferentLeaseHolder (1.82s)
=== RUN   TestStoreRangeSplitRaceUninitializedRHS
I170517 07:33:29.944830 64903 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:29.945138 64903 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:37819" > attrs:<> locality:<>
W170517 07:33:29.976240 64903 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:29.985204 64903 storage/store.go:1252  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I170517 07:33:29.985829 64892 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:37819
I170517 07:33:29.986611 64903 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:60751" > attrs:<> locality:<>
I170517 07:33:30.028608 65113 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 77d68522 at index 14
I170517 07:33:30.031429 65113 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 32, log entries: 4, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:30.038625 65115 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 14 (id=77d68522, encoded size=5778, 1 rocksdb batches, 4 log entries)
I170517 07:33:30.040026 65115 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:30.043597 65113 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:30.053458 65021 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
W170517 07:33:30.147325 64958 vendor/github.com/coreos/etcd/raft/raft.go:793  [s1,r1/1:/M{in-ax}] 1 stepped down to follower since quorum is not active
I170517 07:33:30.186995 65118 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:30.344590 65145 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a\x87\xff" [r2]
I170517 07:33:30.352786 65149 storage/replica_command.go:177  [s1,r1/1:/M{in-ax}] test injecting error: ReadWithinUncertaintyIntervalError: read at time 0.000000123,124 encountered previous write with future timestamp 0.000000123,124 within uncertainty interval
E170517 07:33:30.428144 64921 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a‡ÿ}] range requires a replication change, but lacks a quorum of live replicas (0/2)
E170517 07:33:30.429153 64921 storage/queue.go:634  [replicate,s1,r2/1:{a‡ÿ-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:33:30.430316 64881 storage/replica_command.go:2633  [s1,r1/1:{/Min-a‡ÿ}] initiating a split of this range at key "a\x87\xfe" [r3]
I170517 07:33:30.438366 65203 storage/replica_command.go:177  [s1,r1/1:{/Min-a‡ÿ}] test injecting error: ReadWithinUncertaintyIntervalError: read at time 0.000000123,168 encountered previous write with future timestamp 0.000000123,168 within uncertainty interval
W170517 07:33:30.479162 65008 storage/store.go:3160  [s1] got error from r3, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.479695 65008 storage/store.go:3160  [s1] got error from r3, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.479835 65008 storage/store.go:3160  [s1] got error from r3, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.479914 65008 storage/store.go:3160  [s1] got error from r3, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.479988 65008 storage/store.go:3160  [s1] got error from r3, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.480075 65008 storage/store.go:3160  [s1] got error from r3, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.480163 65008 storage/store.go:3160  [s1] got error from r3, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.480246 65008 storage/store.go:3160  [s1] got error from r3, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.480337 65008 storage/store.go:3160  [s1] got error from r3, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.487893 65008 storage/store.go:3160  [s1] got error from r3, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.488108 65008 storage/store.go:3160  [s1] got error from r3, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.497670 65008 storage/store.go:3160  [s1] got error from r3, replica (n2,s2):2: raft: cannot step as peer not found
E170517 07:33:30.499738 64921 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a‡þ}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:33:30.504053 65152 storage/replica_command.go:2633  [s1,r1/1:{/Min-a‡þ}] initiating a split of this range at key "a\x87\xfd" [r4]
E170517 07:33:30.505956 64921 storage/queue.go:634  [replicate,s1,r3/1:a‡{þ-ÿ}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:33:30.533403 65219 storage/replica_command.go:177  [s1,r1/1:{/Min-a‡þ}] test injecting error: ReadWithinUncertaintyIntervalError: read at time 0.000000123,208 encountered previous write with future timestamp 0.000000123,208 within uncertainty interval
W170517 07:33:30.556587 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.557371 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.557521 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.557753 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.557857 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.557926 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.557991 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.558058 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.558126 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.558202 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.558282 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.558352 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.558413 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.558477 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.558551 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.558635 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.562447 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.562828 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.565786 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.567436 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.567659 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.568027 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.568298 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.568496 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.568684 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.577743 65008 storage/store.go:3160  [s1] got error from r4, replica (n2,s2):2: raft: cannot step as peer not found
E170517 07:33:30.582127 64921 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a‡ý}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:33:30.584218 64780 storage/replica_command.go:2633  [s1,r1/1:{/Min-a‡ý}] initiating a split of this range at key "a\x87\xfc" [r5]
E170517 07:33:30.584755 64921 storage/queue.go:634  [replicate,s1,r4/1:a‡{ý-þ}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:33:30.613985 64783 storage/replica_command.go:177  [s1,r1/1:{/Min-a‡ý}] test injecting error: ReadWithinUncertaintyIntervalError: read at time 0.000000123,256 encountered previous write with future timestamp 0.000000123,256 within uncertainty interval
W170517 07:33:30.619046 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.619223 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.635678 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.635954 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.636072 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.636150 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.636228 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.713320 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.713598 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.713733 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.713849 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.713939 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.714026 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.714115 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.714214 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.714307 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.714392 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.714477 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.714561 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.714650 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.714757 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.714964 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.715071 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.715172 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.715263 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.715431 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.716236 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.716337 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.716427 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.716512 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.716604 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.716710 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.716907 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.716994 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.717110 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.717299 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.717399 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.717487 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.717592 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.717678 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.717757 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.717853 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.717947 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.718059 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.718226 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.718307 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.718400 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.718582 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.718677 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.718758 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.718844 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.718921 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.719011 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.719106 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.719185 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.719272 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.719355 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.719445 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.722159 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.725305 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.726648 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.726768 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.726924 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.727105 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.727213 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.727300 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.727383 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.727508 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.727669 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.727799 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.727893 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.727978 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.728072 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.728156 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.728249 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.728358 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.728462 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.728575 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.732151 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.732354 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.732459 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.732612 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.732813 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.736259 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.736403 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.736506 65008 storage/store.go:3160  [s1] got error from r5, replica (n2,s2):2: raft: cannot step as peer not found
E170517 07:33:30.738700 64921 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a‡ü}] range requires a replication change, but lacks a quorum of live replicas (0/2)
E170517 07:33:30.743336 64921 storage/queue.go:634  [replicate,s1,r5/1:a‡{ü-ý}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:33:30.756589 65189 storage/replica_command.go:2633  [s1,r1/1:{/Min-a‡ü}] initiating a split of this range at key "a\x87\xfb" [r6]
I170517 07:33:30.772427 65191 storage/replica_command.go:177  [s1,r1/1:{/Min-a‡ü}] test injecting error: ReadWithinUncertaintyIntervalError: read at time 0.000000123,344 encountered previous write with future timestamp 0.000000123,344 within uncertainty interval
W170517 07:33:30.779521 65008 storage/store.go:3160  [s1] got error from r6, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.780774 65008 storage/store.go:3160  [s1] got error from r6, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.780929 65008 storage/store.go:3160  [s1] got error from r6, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.791174 65008 storage/store.go:3160  [s1] got error from r6, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.792771 65008 storage/store.go:3160  [s1] got error from r6, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.799536 65008 storage/store.go:3160  [s1] got error from r6, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.817610 65008 storage/store.go:3160  [s1] got error from r6, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.818460 65008 storage/store.go:3160  [s1] got error from r6, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.819239 65008 storage/store.go:3160  [s1] got error from r6, replica (n2,s2):2: raft: cannot step as peer not found
E170517 07:33:30.823708 64921 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a‡û}] range requires a replication change, but lacks a quorum of live replicas (0/2)
E170517 07:33:30.832159 64921 storage/queue.go:634  [replicate,s1,r6/1:a‡{û-ü}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:33:30.833712 65237 storage/replica_command.go:2633  [s1,r1/1:{/Min-a‡û}] initiating a split of this range at key "a\x87\xfa" [r7]
I170517 07:33:30.872725 65253 storage/replica_command.go:177  [s1,r1/1:{/Min-a‡û}] test injecting error: ReadWithinUncertaintyIntervalError: read at time 0.000000123,391 encountered previous write with future timestamp 0.000000123,391 within uncertainty interval
W170517 07:33:30.884348 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.884543 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.884646 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.884720 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.889360 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.890513 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.891687 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.891906 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.893412 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.893851 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.894033 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.894125 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.894239 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.895204 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.897980 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.898162 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.898258 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.904171 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.908887 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.909883 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.910106 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.910241 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.910329 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.910426 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.929156 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.929364 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.929470 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.929562 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.929650 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.929746 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.929843 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.929937 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.930025 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.930131 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.930219 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.930326 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.930419 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.930514 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.930607 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.930692 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.930772 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.930859 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.942901 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.943164 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.943311 65008 storage/store.go:3160  [s1] got error from r7, replica (n2,s2):2: raft: cannot step as peer not found
E170517 07:33:30.945342 64921 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a‡ú}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:33:30.955567 65240 storage/replica_command.go:2633  [s1,r1/1:{/Min-a‡ú}] initiating a split of this range at key "a\x87\xf9" [r8]
E170517 07:33:30.957510 64921 storage/queue.go:634  [replicate,s1,r7/1:a‡{ú-û}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:33:30.963935 65209 storage/replica_command.go:177  [s1,r1/1:{/Min-a‡ú}] test injecting error: ReadWithinUncertaintyIntervalError: read at time 0.000000123,439 encountered previous write with future timestamp 0.000000123,439 within uncertainty interval
W170517 07:33:30.968806 65008 storage/store.go:3160  [s1] got error from r8, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.972644 65008 storage/store.go:3160  [s1] got error from r8, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.983456 65008 storage/store.go:3160  [s1] got error from r8, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.984552 65008 storage/store.go:3160  [s1] got error from r8, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.985126 65008 storage/store.go:3160  [s1] got error from r8, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:30.993285 65008 storage/store.go:3160  [s1] got error from r8, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.008371 65008 storage/store.go:3160  [s1] got error from r8, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.008708 65008 storage/store.go:3160  [s1] got error from r8, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.010616 65008 storage/store.go:3160  [s1] got error from r8, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.011002 65008 storage/store.go:3160  [s1] got error from r8, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.011150 65008 storage/store.go:3160  [s1] got error from r8, replica (n2,s2):2: raft: cannot step as peer not found
E170517 07:33:31.031178 64921 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a‡ù}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:33:31.033437 65241 storage/replica_command.go:2633  [s1,r1/1:{/Min-a‡ù}] initiating a split of this range at key "a\x87\xf8" [r9]
E170517 07:33:31.037885 64921 storage/queue.go:634  [replicate,s1,r8/1:a‡{ù-ú}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:33:31.058960 64897 storage/replica_command.go:177  [s1,r1/1:{/Min-a‡ù}] test injecting error: ReadWithinUncertaintyIntervalError: read at time 0.000000123,480 encountered previous write with future timestamp 0.000000123,480 within uncertainty interval
W170517 07:33:31.125015 65008 storage/store.go:3160  [s1] got error from r9, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.125253 65008 storage/store.go:3160  [s1] got error from r9, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.125416 65008 storage/store.go:3160  [s1] got error from r9, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.125511 65008 storage/store.go:3160  [s1] got error from r9, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.125600 65008 storage/store.go:3160  [s1] got error from r9, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.126362 65008 storage/store.go:3160  [s1] got error from r9, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.131363 65008 storage/store.go:3160  [s1] got error from r9, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.131613 65008 storage/store.go:3160  [s1] got error from r9, replica (n2,s2):2: raft: cannot step as peer not found
E170517 07:33:31.149496 64921 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a‡ø}] range requires a replication change, but lacks a quorum of live replicas (0/2)
W170517 07:33:31.149996 65008 storage/store.go:3160  [s1] got error from r9, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.150268 65008 storage/store.go:3160  [s1] got error from r9, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.150450 65008 storage/store.go:3160  [s1] got error from r9, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.150553 65008 storage/store.go:3160  [s1] got error from r9, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.150654 65008 storage/store.go:3160  [s1] got error from r9, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.150766 65008 storage/store.go:3160  [s1] got error from r9, replica (n2,s2):2: raft: cannot step as peer not found
I170517 07:33:31.150952 65178 storage/replica_command.go:2633  [s1,r1/1:{/Min-a‡ø}] initiating a split of this range at key "a\x87\xf7" [r10]
E170517 07:33:31.152336 64921 storage/queue.go:634  [replicate,s1,r9/1:a‡{ø-ù}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:33:31.175337 65223 storage/replica_command.go:177  [s1,r1/1:{/Min-a‡ø}] test injecting error: ReadWithinUncertaintyIntervalError: read at time 0.000000123,524 encountered previous write with future timestamp 0.000000123,524 within uncertainty interval
W170517 07:33:31.180042 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.188568 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.189710 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.190301 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.190470 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.190592 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.190695 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.191047 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.191524 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.199392 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.199940 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.201175 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.201367 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.201469 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.201553 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.203571 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.213341 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.224867 65008 storage/store.go:3160  [s1] got error from r10, replica (n2,s2):2: raft: cannot step as peer not found
E170517 07:33:31.261156 64921 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a‡÷}] range requires a replication change, but lacks a quorum of live replicas (0/2)
E170517 07:33:31.261735 64921 storage/queue.go:634  [replicate,s1,r10/1:a‡{÷-ø}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:33:31.264036 65163 storage/replica_command.go:2633  [s1,r1/1:{/Min-a‡÷}] initiating a split of this range at key "a\x87\xf6" [r11]
I170517 07:33:31.299137 65242 storage/replica_command.go:177  [s1,r1/1:{/Min-a‡÷}] test injecting error: ReadWithinUncertaintyIntervalError: read at time 0.000000123,576 encountered previous write with future timestamp 0.000000123,576 within uncertainty interval
W170517 07:33:31.308990 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.309142 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.311803 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.312218 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.312332 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.344054 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.344213 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.345073 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.347349 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.347481 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.347561 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.347654 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.347734 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.359666 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.360015 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.366795 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.367446 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.367535 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.370761 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.371869 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
E170517 07:33:31.373838 64921 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a‡ö}] range requires a replication change, but lacks a quorum of live replicas (0/2)
E170517 07:33:31.374654 64921 storage/queue.go:634  [replicate,s1,r11/1:a‡{ö-÷}] range requires a replication change, but lacks a quorum of live replicas (0/2)
I170517 07:33:31.374846 65180 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/queue.go:523
W170517 07:33:31.375321 65008 storage/store.go:3160  [s1] got error from r11, replica (n2,s2):2: raft: cannot step as peer not found
W170517 07:33:31.383050 65138 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n1,s1):1
W170517 07:33:31.383456 65138 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n1,s1):1
W170517 07:33:31.384890 65120 storage/store.go:3156  [s2] raft error: node 1 claims to not contain store 1 for replica (n1,s1):1: store 1 was not found
W170517 07:33:31.385131 65118 storage/raft_transport.go:442  raft transport stream to node 1 failed: store 1 was not found
I170517 07:33:31.389341 65212 storage/raft_transport.go:436  raft transport stream to node 1 established
W170517 07:33:31.391331 65268 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):?: no handler registered for (n1,s1):?
W170517 07:33:31.398152 65268 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n1,s1):1
W170517 07:33:31.398412 65268 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n1,s1):1
W170517 07:33:31.398514 65008 storage/raft_transport.go:476  no handler found for store 1 in response range_id:11 from_replica:<node_id:2 store_id:2 replica_id:2 > to_replica:<node_id:1 store_id:1 replica_id:1 > union:<error:<message:"raft: cannot step as peer not found" transaction_restart:NONE origin_node:0 now:<wall_time:0 logical:0 > > >
W170517 07:33:31.399414 65214 storage/store.go:3156  [s2] raft error: node 1 claims to not contain store 1 for replica (n1,s1):?: store 1 was not found
I170517 07:33:31.417084 65014 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:60751->127.0.0.1:55267: use of closed network connection
I170517 07:33:31.417495 64859 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:37819->127.0.0.1:35985: use of closed network connection
--- PASS: TestStoreRangeSplitRaceUninitializedRHS (1.50s)
=== RUN   TestLeaderAfterSplit
I170517 07:33:31.474653 65195 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:31.475050 65195 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:43345" > attrs:<> locality:<>
W170517 07:33:31.506873 65195 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:31.510116 65396 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:43345
I170517 07:33:31.539991 65195 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:44325" > attrs:<> locality:<>
W170517 07:33:31.559458 65195 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:33:31.561315 65402 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:43345
I170517 07:33:31.609346 65195 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:57302" > attrs:<> locality:<>
I170517 07:33:31.626473 65626 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 62bb6b2b at index 15
I170517 07:33:31.647496 65626 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 8ms
I170517 07:33:31.648572 65287 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=62bb6b2b, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:33:31.650108 65287 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:31.652504 65626 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:31.667151 65291 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:31.675772 65640 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot dd01e599 at index 17
I170517 07:33:31.681741 65640 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 5ms
I170517 07:33:31.683044 65295 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=dd01e599, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:33:31.684621 65295 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:33:31.687237 65640 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:31.692730 65632 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:31.707497 65668 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:33:31.881113 65578 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "m" [r2]
W170517 07:33:31.953761 65651 storage/raft_transport.go:257  unable to accept Raft message from (n2,s2):2: no handler registered for (n1,s1):1
W170517 07:33:31.954608 65666 storage/store.go:3156  [s2] raft error: node 1 claims to not contain store 1 for replica (n1,s1):1: store 1 was not found
W170517 07:33:31.954884 65632 storage/raft_transport.go:442  raft transport stream to node 1 failed: store 1 was not found
I170517 07:33:31.959335 65684 storage/raft_transport.go:436  raft transport stream to node 1 established
W170517 07:33:31.968353 65700 storage/raft_transport.go:257  unable to accept Raft message from (n3,s3):?: no handler registered for (n1,s1):?
W170517 07:33:31.969107 65652 storage/store.go:3156  [s3] raft error: node 1 claims to not contain store 1 for replica (n1,s1):?: store 1 was not found
I170517 07:33:31.974157 65403 storage/raft_transport.go:436  raft transport stream to node 1 established
W170517 07:33:31.975124 65390 storage/raft_transport.go:257  unable to accept Raft message from (n3,s3):3: no handler registered for (n1,s1):1
W170517 07:33:31.982916 65405 storage/raft_transport.go:476  no handler found for store 3 in response range_id:2 from_replica:<node_id:1 store_id:1 replica_id:1 > to_replica:<node_id:3 store_id:3 replica_id:3 > union:<error:<message:"store 1 was not found" transaction_restart:NONE origin_node:0 detail:<store_not_found:<store_id:1 > > now:<wall_time:0 logical:0 > > >
--- PASS: TestLeaderAfterSplit (0.58s)
=== RUN   TestStoreSplitBeginTxnPushMetaIntentRace
I170517 07:33:32.011963 65391 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:32.068424 65391 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key /System/"" [r2]
I170517 07:33:32.098379 65812 storage/replica_command.go:2633  [s1,r2/1:/{System/-Max}] initiating a split of this range at key "a" [r3]
I170517 07:33:32.106574 65812 storage/replica_command.go:177  [s1,r2/1:/{System/-Max}] test injecting error: node unavailable; try another peer
I170517 07:33:32.169299 65391 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
W170517 07:33:32.173510 65800 storage/replica.go:2296  [s1,r2/1:{/System/-a}] shutdown cancellation after 0.0s of attempting command GC [/System/"",/System/"\x00")
W170517 07:33:32.173902 65800 storage/intent_resolver.go:370  could not GC completed transaction anchored at /Local/Range/"\x04"/RangeDescriptor: result is ambiguous (server shutdown)
--- PASS: TestStoreSplitBeginTxnPushMetaIntentRace (0.20s)
=== RUN   TestStorePushTxnQueueEnabledOnSplit
I170517 07:33:33.372479 65955 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:33.409181 65955 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:33.478474 65955 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key /Table/50 [r2]
I170517 07:33:33.495660 65955 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/queue.go:693
1      kv/txn_coord_sender.go:981
I170517 07:33:33.495964 65955 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/queue.go:693
--- PASS: TestStorePushTxnQueueEnabledOnSplit (0.21s)
=== RUN   TestDistributedTxnCleanup
I170517 07:33:33.581871 65945 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:33.615466 65945 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:33.630277 65945 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r2]
I170517 07:33:33.646584 65945 storage/replica_command.go:2633  [s1,r2/1:{a-/Max}] initiating a split of this range at key "b" [r3]
W170517 07:33:33.661730 65945 storage/stores.go:218  range not contained in one range: [/Meta2/"b","b\x00"), but have [/Min,"a")
=== RUN   TestDistributedTxnCleanup/force=true,commit=true
W170517 07:33:33.695469 66180 storage/stores.go:218  range not contained in one range: ["a.force=true,commit=true","b.force=true,commit=true\x00"), but have ["a","b")
=== RUN   TestDistributedTxnCleanup/force=true,commit=false
=== RUN   TestDistributedTxnCleanup/force=false,commit=true
=== RUN   TestDistributedTxnCleanup/force=false,commit=false
I170517 07:33:33.825070 65945 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
--- PASS: TestDistributedTxnCleanup (0.30s)
    --- PASS: TestDistributedTxnCleanup/force=true,commit=true (0.05s)
    --- PASS: TestDistributedTxnCleanup/force=true,commit=false (0.04s)
    --- PASS: TestDistributedTxnCleanup/force=false,commit=true (0.02s)
    --- PASS: TestDistributedTxnCleanup/force=false,commit=false (0.03s)
=== RUN   TestUnsplittableRange
I170517 07:33:33.879240 66181 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:33.893734 66181 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:33.903821 66212 storage/split_queue.go:92  [split,s1,r1/1:/M{in-ax}] splitting at key /System/""
I170517 07:33:33.933421 66212 storage/replica_command.go:2633  [split,s1,r1/1:/M{in-ax}] initiating a split of this range at key /System/"" [r2]
I170517 07:33:33.957525 66212 storage/split_queue.go:92  [split,s1,r2/1:/{System/-Max}] splitting at key /System/tsd
I170517 07:33:33.957744 66212 storage/replica_command.go:2633  [split,s1,r2/1:/{System/-Max}] initiating a split of this range at key /System/tsd [r3]
W170517 07:33:33.961994 66212 storage/stores.go:218  range not contained in one range: [/Meta2/System/tsd,/System/tsd/NULL), but have [/Min,/System/"")
I170517 07:33:34.016246 66181 storage/split_queue.go:92  [split,s1,r3/1:/{System/tsd-Max}] splitting at key /System/"tse"
I170517 07:33:34.016462 66181 storage/replica_command.go:2633  [split,s1,r3/1:/{System/tsd-Max}] initiating a split of this range at key /System/"tse" [r4]
I170517 07:33:34.064504 66212 storage/split_queue.go:92  [split,s1,r4/1:/{System/tse-Max}] splitting at key /Table/0/0
I170517 07:33:34.064680 66212 storage/replica_command.go:2633  [split,s1,r4/1:/{System/tse-Max}] initiating a split of this range at key /Table/0 [r5]
I170517 07:33:34.103202 66212 storage/split_queue.go:92  [split,s1,r5/1:/{Table/0-Max}] splitting at key /Table/11/0
I170517 07:33:34.103442 66212 storage/replica_command.go:2633  [split,s1,r5/1:/{Table/0-Max}] initiating a split of this range at key /Table/11 [r6]
I170517 07:33:34.130604 66212 storage/split_queue.go:92  [split,s1,r6/1:/{Table/11-Max}] splitting at key /Table/12/0
I170517 07:33:34.130782 66212 storage/replica_command.go:2633  [split,s1,r6/1:/{Table/11-Max}] initiating a split of this range at key /Table/12 [r7]
I170517 07:33:34.165674 66212 storage/split_queue.go:92  [split,s1,r7/1:/{Table/12-Max}] splitting at key /Table/13/0
I170517 07:33:34.165837 66212 storage/replica_command.go:2633  [split,s1,r7/1:/{Table/12-Max}] initiating a split of this range at key /Table/13 [r8]
I170517 07:33:34.188783 66212 storage/split_queue.go:92  [split,s1,r8/1:/{Table/13-Max}] splitting at key /Table/14/0
I170517 07:33:34.188967 66212 storage/replica_command.go:2633  [split,s1,r8/1:/{Table/13-Max}] initiating a split of this range at key /Table/14 [r9]
I170517 07:33:34.256658 66181 storage/split_queue.go:111  [split,s1,r9/1:/{Table/14-Max}] splitting size=65557 max=65536
I170517 07:33:34.276895 66212 storage/split_queue.go:111  [split,s1,r9/1:/{Table/14-Max}] splitting size=196671 max=131114
I170517 07:33:34.277463 66212 storage/replica_command.go:2633  [split,s1,r9/1:/{Table/14-Max}] initiating a split of this range at key /Table/14/2 [r10]
I170517 07:33:34.327418 66181 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
W170517 07:33:34.327618 66355 storage/intent_resolver.go:327  [n1,s1,r9/1:/Table/14{-/2}]: failed to resolve intents: node unavailable; try another peer
--- PASS: TestUnsplittableRange (0.47s)
=== RUN   TestPushTxnQueueDependencyCycleWithRangeSplit
=== RUN   TestPushTxnQueueDependencyCycleWithRangeSplit/read-2nd-pass:false
I170517 07:33:34.351523 66335 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:34.397376 66335 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:34.414625 66335 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r2]
I170517 07:33:34.443510 66335 storage/replica_command.go:2633  [s1,r2/1:{a-/Max}] initiating a split of this range at key "b" [r3]
W170517 07:33:34.451426 66335 storage/stores.go:218  range not contained in one range: [/Meta2/"b","b\x00"), but have [/Min,"a")
I170517 07:33:34.600793 66335 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
1      kv/txn_coord_sender.go:981
W170517 07:33:34.604074 66467 storage/intent_resolver.go:327  [n1,s1,r2/1:{a-b}]: failed to resolve intents: node unavailable; try another peer
I170517 07:33:34.604283 66335 util/stop/stopper.go:505  quiescing; tasks left:
1      storage/intent_resolver.go:259
=== RUN   TestPushTxnQueueDependencyCycleWithRangeSplit/read-2nd-pass:true
I170517 07:33:34.618198 66312 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"" address_field:"" > attrs:<> locality:<>
I170517 07:33:34.636447 66312 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:34.650654 66312 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r2]
I170517 07:33:34.701391 66312 storage/replica_command.go:2633  [s1,r2/1:{a-/Max}] initiating a split of this range at key "b" [r3]
W170517 07:33:34.705318 66312 storage/stores.go:218  range not contained in one range: [/Meta2/"b","b\x00"), but have [/Min,"a")
--- PASS: TestPushTxnQueueDependencyCycleWithRangeSplit (0.49s)
    --- PASS: TestPushTxnQueueDependencyCycleWithRangeSplit/read-2nd-pass:false (0.27s)
    --- PASS: TestPushTxnQueueDependencyCycleWithRangeSplit/read-2nd-pass:true (0.21s)
=== RUN   TestComputeStatsForKeySpan
I170517 07:33:34.861619 66567 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:34.861923 66567 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:33064" > attrs:<> locality:<>
W170517 07:33:34.890321 66567 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:34.893791 66682 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:33064
I170517 07:33:34.908780 66567 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:57471" > attrs:<> locality:<>
W170517 07:33:34.938493 66567 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:33:34.941352 66866 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:33064
I170517 07:33:34.953789 66567 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:45209" > attrs:<> locality:<>
I170517 07:33:34.984204 66567 storage/replica_command.go:2633  [s1,r1/1:/M{in-ax}] initiating a split of this range at key "a" [r2]
E170517 07:33:34.997148 66366 storage/queue.go:634  [replicate,s1,r1/1:{/Min-a}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:33:34.997848 66366 storage/queue.go:634  [replicate,s1,r2/1:{a-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:33:34.998700 66567 storage/replica_command.go:2633  [s1,r2/1:{a-/Max}] initiating a split of this range at key "c" [r3]
W170517 07:33:35.003237 66959 storage/stores.go:218  range not contained in one range: [/Meta2/"c","c\x00"), but have [/Min,"a")
E170517 07:33:35.030538 66366 storage/queue.go:634  [replicate,s1,r2/1:{a-c}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:33:35.031044 66366 storage/queue.go:634  [replicate,s1,r3/1:{c-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:33:35.037170 66567 storage/replica_command.go:2633  [s1,r3/1:{c-/Max}] initiating a split of this range at key "e" [r4]
E170517 07:33:35.098005 66366 storage/queue.go:634  [replicate,s1,r3/1:{c-e}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:33:35.098565 66366 storage/queue.go:634  [replicate,s1,r4/1:{e-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:33:35.100047 66567 storage/replica_command.go:2633  [s1,r4/1:{e-/Max}] initiating a split of this range at key "g" [r5]
E170517 07:33:35.119101 66366 storage/queue.go:634  [replicate,s1,r4/1:{e-g}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:33:35.119682 66366 storage/queue.go:634  [replicate,s1,r5/1:{g-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:33:35.123752 66567 storage/replica_command.go:2633  [s1,r5/1:{g-/Max}] initiating a split of this range at key "i" [r6]
E170517 07:33:35.161728 66366 storage/queue.go:634  [replicate,s1,r5/1:{g-i}] range requires a replication change, but lacks a quorum of live replicas (0/1)
E170517 07:33:35.162226 66366 storage/queue.go:634  [replicate,s1,r6/1:{i-/Max}] range requires a replication change, but lacks a quorum of live replicas (0/1)
I170517 07:33:35.290695 66689 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:45209->127.0.0.1:53119: use of closed network connection
I170517 07:33:35.290870 66758 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:33:35.291217 65695 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:57471->127.0.0.1:41320: use of closed network connection
I170517 07:33:35.293621 66458 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:33064->127.0.0.1:57103: read: connection reset by peer
I170517 07:33:35.302384 66761 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:45209: getsockopt: connection reset by peer"; Reconnecting to {127.0.0.1:45209 <nil>}
I170517 07:33:35.302578 66761 vendor/google.golang.org/grpc/clientconn.go:906  grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
--- PASS: TestComputeStatsForKeySpan (0.49s)
=== RUN   TestSortRangeDescByAge
--- PASS: TestSortRangeDescByAge (0.02s)
=== RUN   TestConsistencyQueueRequiresLive
I170517 07:33:35.412137 67001 storage/store.go:1252  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:35.412478 67001 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:40192" > attrs:<> locality:<>
W170517 07:33:35.450066 67001 gossip/gossip.go:1196  [n2] no incoming or outgoing connections
I170517 07:33:35.453854 67019 gossip/client.go:131  [n2] started gossip client to 127.0.0.1:40192
I170517 07:33:35.471027 67001 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:51379" > attrs:<> locality:<>
W170517 07:33:35.493565 67001 gossip/gossip.go:1196  [n3] no incoming or outgoing connections
I170517 07:33:35.495234 67111 gossip/client.go:131  [n3] started gossip client to 127.0.0.1:40192
I170517 07:33:35.515212 67001 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:54250" > attrs:<> locality:<>
I170517 07:33:35.532950 67334 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 715abe3e at index 15
I170517 07:33:35.537105 66721 storage/replica_raftstorage.go:597  [s2,r1/?:{-}] applying preemptive snapshot at index 15 (id=715abe3e, encoded size=5574, 1 rocksdb batches, 5 log entries)
I170517 07:33:35.538815 66721 storage/replica_raftstorage.go:605  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=0ms commit=0ms]
I170517 07:33:35.539870 67334 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 33, log entries: 5, rate-limit: 2.0 MiB/sec, 6ms
I170517 07:33:35.541783 67334 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:35.547384 67363 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:35.553823 67381 storage/replica_raftstorage.go:416  [s1,r1/1:/M{in-ax}] generated preemptive snapshot 29ff3980 at index 17
I170517 07:33:35.556831 67381 storage/store.go:3345  [s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 36, log entries: 7, rate-limit: 2.0 MiB/sec, 2ms
I170517 07:33:35.558561 67384 storage/replica_raftstorage.go:597  [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=29ff3980, encoded size=7121, 1 rocksdb batches, 7 log entries)
I170517 07:33:35.560177 67384 storage/replica_raftstorage.go:605  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 1ms [clear=0ms batch=0ms entries=1ms commit=0ms]
I170517 07:33:35.562897 67381 storage/replica_command.go:3538  [s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:35.571794 67366 storage/raft_transport.go:436  raft transport stream to node 1 established
I170517 07:33:35.585933 67370 storage/replica.go:2590  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:33:35.676197 67001 storage/client_test.go:1173  test clock advanced to: 1.800000125,0
W170517 07:33:35.691869 67361 storage/raft_transport.go:442  raft transport stream to node 3 failed: EOF
I170517 07:33:35.693675 67108 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:51379->127.0.0.1:42096: use of closed network connection
I170517 07:33:35.694083 66478 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:40192->127.0.0.1:50264: use of closed network connection
W170517 07:33:35.695747 67366 storage/raft_transport.go:442  raft transport stream to node 1 failed: rpc error: code = Internal desc = transport is closing
--- PASS: TestConsistencyQueueRequiresLive (0.36s)
=== RUN   TestGossipFirstRange
W170517 07:33:35.752249 67339 server/status/runtime.go:111  Could not parse build timestamp: parsing time "" as "2006/01/02 15:04:05": cannot parse "" as "2006"
I170517 07:33:35.757394 67339 server/config.go:436  1 storage engine initialized
I170517 07:33:35.759209 67339 server/node.go:454  [n?] store [n0,s0] not bootstrapped
I170517 07:33:35.822584 67339 server/node.go:389  [n?] **** cluster 04e79f7f-df5c-405d-8fae-23c92736960a has been created
I170517 07:33:35.822708 67339 server/node.go:390  [n?] **** add additional nodes by specifying --join=127.0.0.1:45912
I170517 07:33:35.847167 67339 storage/store.go:1252  [n1] [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:35.848222 67339 server/node.go:467  [n1] initialized store [n1,s1]: {Capacity:536870912 Available:536870912 RangeCount:1 LeaseCount:0}
I170517 07:33:35.848445 67339 server/node.go:351  [n1] node ID 1 initialized
I170517 07:33:35.848701 67339 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:45912" > attrs:<> locality:<>
I170517 07:33:35.851472 67339 storage/stores.go:296  [n1] read 0 node addresses from persistent storage
I170517 07:33:35.852008 67339 server/node.go:608  [n1] connecting to gossip network to verify cluster ID...
I170517 07:33:35.852157 67339 server/node.go:633  [n1] node connected via gossip and verified as part of cluster "04e79f7f-df5c-405d-8fae-23c92736960a"
I170517 07:33:35.852467 67339 server/node.go:405  [n1] node=1: started with [[]=] engine(s) and attributes []
I170517 07:33:35.854466 67339 sql/executor.go:349  [n1] creating distSQLPlanner with address {tcp 127.0.0.1:45912}
I170517 07:33:35.889786 67339 server/server.go:736  [n1] starting https server at 127.0.0.1:48952
I170517 07:33:35.889957 67339 server/server.go:737  [n1] starting grpc/postgres server at 127.0.0.1:45912
I170517 07:33:35.890001 67339 server/server.go:738  [n1] advertising CockroachDB node at 127.0.0.1:45912
I170517 07:33:35.963005 67339 sql/event_log.go:101  [n1] Event: "alter_table", target: 12, info: {TableName:eventlog Statement:ALTER TABLE system.eventlog ALTER COLUMN uniqueid SET DEFAULT uuid_v4() User:node MutationID:0 CascadeDroppedViews:[]}
I170517 07:33:36.048956 67339 sql/lease.go:372  [n1] publish: descID=12 (eventlog) version=2 mtime=2017-05-17 07:33:36.048869163 +0000 UTC
I170517 07:33:36.235951 67339 server/server.go:848  [n1] done ensuring all necessary migrations have run
I170517 07:33:36.236117 67339 server/server.go:850  [n1] serving sql connections
I170517 07:33:36.253023 67718 sql/event_log.go:101  [n1] Event: "node_join", target: 1, info: {Descriptor:{NodeID:1 Address:{NetworkField:tcp AddressField:127.0.0.1:45912} Attrs: Locality:} ClusterID:04e79f7f-df5c-405d-8fae-23c92736960a StartedAt:1495006415852202274 LastUp:1495006415852202274}
W170517 07:33:36.348812 67339 server/status/runtime.go:111  Could not parse build timestamp: parsing time "" as "2006/01/02 15:04:05": cannot parse "" as "2006"
W170517 07:33:36.351083 67339 gossip/gossip.go:1196  [n?] no incoming or outgoing connections
I170517 07:33:36.357756 67339 server/config.go:436  1 storage engine initialized
I170517 07:33:36.363108 67339 server/node.go:454  [n?] store [n0,s0] not bootstrapped
I170517 07:33:36.363298 67339 storage/stores.go:296  [n?] read 0 node addresses from persistent storage
I170517 07:33:36.363648 67339 server/node.go:608  [n?] connecting to gossip network to verify cluster ID...
I170517 07:33:36.471761 67707 gossip/client.go:131  [n?] started gossip client to 127.0.0.1:45912
I170517 07:33:36.473134 67780 gossip/server.go:234  [n1] received initial cluster-verification connection from {tcp 127.0.0.1:43118}
I170517 07:33:36.475459 67339 server/node.go:633  [n?] node connected via gossip and verified as part of cluster "04e79f7f-df5c-405d-8fae-23c92736960a"
I170517 07:33:36.477361 67781 storage/stores.go:312  [n?] wrote 1 node addresses to persistent storage
I170517 07:33:36.488351 67339 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
I170517 07:33:36.501398 67339 server/node.go:344  [n?] new node allocated ID 2
I170517 07:33:36.501692 67339 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:43118" > attrs:<> locality:<>
I170517 07:33:36.502983 67339 server/node.go:405  [n2] node=2: started with [[]=] engine(s) and attributes []
I170517 07:33:36.514733 67749 storage/stores.go:312  [n1] wrote 1 node addresses to persistent storage
I170517 07:33:36.536138 67339 sql/executor.go:349  [n2] creating distSQLPlanner with address {tcp 127.0.0.1:43118}
I170517 07:33:36.604348 67793 server/node.go:589  [n2] bootstrapped store [n2,s2]
I170517 07:33:36.633620 67339 server/server.go:736  [n2] starting https server at 127.0.0.1:46748
I170517 07:33:36.633739 67339 server/server.go:737  [n2] starting grpc/postgres server at 127.0.0.1:43118
I170517 07:33:36.633812 67339 server/server.go:738  [n2] advertising CockroachDB node at 127.0.0.1:43118
I170517 07:33:36.639287 67339 server/server.go:848  [n2] done ensuring all necessary migrations have run
I170517 07:33:36.639410 67339 server/server.go:850  [n2] serving sql connections
I170517 07:33:36.717646 67939 sql/event_log.go:101  [n2] Event: "node_join", target: 2, info: {Descriptor:{NodeID:2 Address:{NetworkField:tcp AddressField:127.0.0.1:43118} Attrs: Locality:} ClusterID:04e79f7f-df5c-405d-8fae-23c92736960a StartedAt:1495006416502742313 LastUp:1495006416502742313}
W170517 07:33:36.770163 67339 server/status/runtime.go:111  Could not parse build timestamp: parsing time "" as "2006/01/02 15:04:05": cannot parse "" as "2006"
W170517 07:33:36.771902 67339 gossip/gossip.go:1196  [n?] no incoming or outgoing connections
I170517 07:33:36.773073 67339 server/config.go:436  1 storage engine initialized
I170517 07:33:36.774659 67339 server/node.go:454  [n?] store [n0,s0] not bootstrapped
I170517 07:33:36.775821 67339 storage/stores.go:296  [n?] read 0 node addresses from persistent storage
I170517 07:33:36.775949 67339 server/node.go:608  [n?] connecting to gossip network to verify cluster ID...
I170517 07:33:36.829330 67917 gossip/client.go:131  [n?] started gossip client to 127.0.0.1:45912
I170517 07:33:36.831796 67964 gossip/server.go:234  [n1] received initial cluster-verification connection from {tcp 127.0.0.1:51189}
I170517 07:33:36.839946 67932 storage/stores.go:312  [n?] wrote 1 node addresses to persistent storage
I170517 07:33:36.840527 67339 server/node.go:633  [n?] node connected via gossip and verified as part of cluster "04e79f7f-df5c-405d-8fae-23c92736960a"
I170517 07:33:36.847491 67934 storage/stores.go:312  [n?] wrote 2 node addresses to persistent storage
I170517 07:33:36.851800 67339 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
I170517 07:33:36.856162 67339 server/node.go:344  [n?] new node allocated ID 3
I170517 07:33:36.856409 67339 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:51189" > attrs:<> locality:<>
I170517 07:33:36.869237 67339 server/node.go:405  [n3] node=3: started with [[]=] engine(s) and attributes []
I170517 07:33:36.889338 67980 storage/stores.go:312  [n1] wrote 2 node addresses to persistent storage
I170517 07:33:36.891546 67981 storage/stores.go:312  [n2] wrote 2 node addresses to persistent storage
I170517 07:33:36.892616 67339 sql/executor.go:349  [n3] creating distSQLPlanner with address {tcp 127.0.0.1:51189}
I170517 07:33:36.933015 67339 server/server.go:736  [n3] starting https server at 127.0.0.1:46534
I170517 07:33:36.933195 67339 server/server.go:737  [n3] starting grpc/postgres server at 127.0.0.1:51189
I170517 07:33:36.933238 67339 server/server.go:738  [n3] advertising CockroachDB node at 127.0.0.1:51189
I170517 07:33:36.954317 68019 server/node.go:589  [n3] bootstrapped store [n3,s3]
I170517 07:33:36.968289 67339 server/server.go:848  [n3] done ensuring all necessary migrations have run
I170517 07:33:36.968423 67339 server/server.go:850  [n3] serving sql connections
I170517 07:33:37.010432 68164 storage/replica_raftstorage.go:416  [n1,s1,r1/1:/M{in-ax}] generated preemptive snapshot 4a6a3165 at index 59
I170517 07:33:37.069356 67982 sql/event_log.go:101  [n3] Event: "node_join", target: 3, info: {Descriptor:{NodeID:3 Address:{NetworkField:tcp AddressField:127.0.0.1:51189} Attrs: Locality:} ClusterID:04e79f7f-df5c-405d-8fae-23c92736960a StartedAt:1495006416868691758 LastUp:1495006416868691758}
I170517 07:33:37.226549 68164 storage/store.go:3345  [n1,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 719, log entries: 5, rate-limit: 2.0 MiB/sec, 23ms
I170517 07:33:37.228727 68227 storage/replica_raftstorage.go:597  [n2,s2,r1/?:{-}] applying preemptive snapshot at index 59 (id=4a6a3165, encoded size=125926, 1 rocksdb batches, 5 log entries)
I170517 07:33:37.232356 68227 storage/replica_raftstorage.go:605  [n2,s2,r1/?:/M{in-ax}] applied preemptive snapshot in 3ms [clear=0ms batch=0ms entries=1ms commit=1ms]
I170517 07:33:37.240802 68164 storage/replica_command.go:3538  [n1,s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:37.257337 68239 storage/replica.go:2590  [n1,s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:37.277141 68249 storage/raft_transport.go:436  [n2] raft transport stream to node 1 established
I170517 07:33:38.346782 68174 storage/replica_raftstorage.go:416  [n1,s1,r1/1:/M{in-ax}] generated preemptive snapshot 689436ce at index 67
I170517 07:33:38.522073 68174 storage/store.go:3345  [n1,s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 978, log entries: 13, rate-limit: 2.0 MiB/sec, 49ms
I170517 07:33:38.524992 68253 storage/replica_raftstorage.go:597  [n3,s3,r1/?:{-}] applying preemptive snapshot at index 67 (id=689436ce, encoded size=207859, 1 rocksdb batches, 13 log entries)
I170517 07:33:38.529025 68253 storage/replica_raftstorage.go:605  [n3,s3,r1/?:/M{in-ax}] applied preemptive snapshot in 4ms [clear=0ms batch=0ms entries=2ms commit=1ms]
I170517 07:33:38.534419 68174 storage/replica_command.go:3538  [n1,s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:38.558972 68290 storage/replica.go:2590  [n1,s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:33:39.601896 68256 storage/raft_transport.go:436  [n3] raft transport stream to node 1 established
I170517 07:33:39.689981 68286 storage/replica_command.go:3538  [n2,s2,r1/2:/M{in-ax}] change replicas (REMOVE_REPLICA (n1,s1):1): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4]
I170517 07:33:39.791433 68271 storage/replica.go:2590  [n2,s2,r1/2:/M{in-ax}] proposing REMOVE_REPLICA (n1,s1):1: [(n3,s3):3 (n2,s2):2]
I170517 07:33:39.831798 67602 storage/store.go:2139  [replicaGC,n1,s1,r1/1:/M{in-ax}] removing replica
I170517 07:33:39.832700 67602 storage/replica.go:684  [replicaGC,n1,s1,r1/1:/M{in-ax}] removed 85 (75+10) keys in 0ms [clear=0ms commit=0ms]
I170517 07:33:39.885388 68386 util/stop/stopper.go:505  quiescing; tasks left:
3      gossip/infostore.go:301
W170517 07:33:39.890529 68249 storage/raft_transport.go:442  [n2] raft transport stream to node 1 failed: rpc error: code = Internal desc = transport is closing
W170517 07:33:39.894876 68256 storage/raft_transport.go:442  [n3] raft transport stream to node 1 failed: rpc error: code = Internal desc = transport is closing
I170517 07:33:39.895360 66880 kv/transport_race.go:71  transport race promotion: ran 79 iterations on up to 147 requests
--- PASS: TestGossipFirstRange (4.21s)
=== RUN   TestGossipHandlesReplacedNode
W170517 07:33:39.932009 68389 server/status/runtime.go:111  Could not parse build timestamp: parsing time "" as "2006/01/02 15:04:05": cannot parse "" as "2006"
I170517 07:33:39.935633 68389 server/config.go:436  1 storage engine initialized
I170517 07:33:39.936948 68389 server/node.go:454  [n?] store [n0,s0] not bootstrapped
I170517 07:33:39.962712 68389 server/node.go:389  [n?] **** cluster 403e3e46-9202-4288-841d-fbf0bbff3d4c has been created
I170517 07:33:39.962802 68389 server/node.go:390  [n?] **** add additional nodes by specifying --join=127.87.51.24:41559
I170517 07:33:39.972490 68389 storage/store.go:1252  [n1] [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I170517 07:33:39.992915 68389 server/node.go:467  [n1] initialized store [n1,s1]: {Capacity:536870912 Available:536870912 RangeCount:1 LeaseCount:1}
I170517 07:33:39.993143 68389 server/node.go:351  [n1] node ID 1 initialized
I170517 07:33:39.993402 68389 gossip/gossip.go:297  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.87.51.24:41559" > attrs:<> locality:<>
I170517 07:33:39.994063 68389 storage/stores.go:296  [n1] read 0 node addresses from persistent storage
I170517 07:33:39.994529 68389 server/node.go:608  [n1] connecting to gossip network to verify cluster ID...
I170517 07:33:39.994679 68389 server/node.go:633  [n1] node connected via gossip and verified as part of cluster "403e3e46-9202-4288-841d-fbf0bbff3d4c"
I170517 07:33:39.994993 68389 server/node.go:405  [n1] node=1: started with [[]=] engine(s) and attributes []
I170517 07:33:40.007625 68389 sql/executor.go:349  [n1] creating distSQLPlanner with address {tcp 127.87.51.24:41559}
I170517 07:33:40.063967 68389 server/server.go:736  [n1] starting http server at 127.87.51.24:56315
I170517 07:33:40.064080 68389 server/server.go:737  [n1] starting grpc/postgres server at 127.87.51.24:41559
I170517 07:33:40.064131 68389 server/server.go:738  [n1] advertising CockroachDB node at 127.87.51.24:41559
I170517 07:33:40.132180 68389 sql/event_log.go:101  [n1] Event: "alter_table", target: 12, info: {TableName:eventlog Statement:ALTER TABLE system.eventlog ALTER COLUMN uniqueid SET DEFAULT uuid_v4() User:node MutationID:0 CascadeDroppedViews:[]}
I170517 07:33:40.212282 68389 sql/lease.go:372  [n1] publish: descID=12 (eventlog) version=2 mtime=2017-05-17 07:33:40.212148904 +0000 UTC
I170517 07:33:40.384839 68389 server/server.go:848  [n1] done ensuring all necessary migrations have run
I170517 07:33:40.385008 68389 server/server.go:850  [n1] serving sql connections
I170517 07:33:40.446518 68653 sql/event_log.go:101  [n1] Event: "node_join", target: 1, info: {Descriptor:{NodeID:1 Address:{NetworkField:tcp AddressField:127.87.51.24:41559} Attrs: Locality:} ClusterID:403e3e46-9202-4288-841d-fbf0bbff3d4c StartedAt:1495006419994719284 LastUp:1495006419994719284}
W170517 07:33:40.449083 68389 server/status/runtime.go:111  Could not parse build timestamp: parsing time "" as "2006/01/02 15:04:05": cannot parse "" as "2006"
W170517 07:33:40.450704 68389 gossip/gossip.go:1196  [n?] no incoming or outgoing connections
I170517 07:33:40.469694 68684 gossip/server.go:234  [n1] received initial cluster-verification connection from {tcp 127.87.51.24:35673}
I170517 07:33:40.471301 68722 gossip/client.go:131  [n?] started gossip client to 127.87.51.24:41559
I170517 07:33:40.489059 68389 server/config.go:436  1 storage engine initialized
I170517 07:33:40.493354 68389 server/node.go:454  [n?] store [n0,s0] not bootstrapped
I170517 07:33:40.493469 68389 storage/stores.go:296  [n?] read 0 node addresses from persistent storage
I170517 07:33:40.493622 68389 storage/stores.go:312  [n?] wrote 1 node addresses to persistent storage
I170517 07:33:40.493669 68389 server/node.go:608  [n?] connecting to gossip network to verify cluster ID...
I170517 07:33:40.493763 68389 server/node.go:633  [n?] node connected via gossip and verified as part of cluster "403e3e46-9202-4288-841d-fbf0bbff3d4c"
I170517 07:33:40.500478 68389 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
I170517 07:33:40.504487 68389 server/node.go:344  [n?] new node allocated ID 2
I170517 07:33:40.504788 68389 gossip/gossip.go:297  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.87.51.24:35673" > attrs:<> locality:<>
I170517 07:33:40.505608 68389 server/node.go:405  [n2] node=2: started with [[]=] engine(s) and attributes []
I170517 07:33:40.507715 68694 storage/stores.go:312  [n1] wrote 1 node addresses to persistent storage
I170517 07:33:40.534732 68389 sql/executor.go:349  [n2] creating distSQLPlanner with address {tcp 127.87.51.24:35673}
I170517 07:33:40.591612 68672 server/node.go:589  [n2] bootstrapped store [n2,s2]
I170517 07:33:40.646342 68389 server/server.go:736  [n2] starting http server at 127.87.51.24:45130
I170517 07:33:40.647148 68389 server/server.go:737  [n2] starting grpc/postgres server at 127.87.51.24:35673
I170517 07:33:40.647475 68389 server/server.go:738  [n2] advertising CockroachDB node at 127.87.51.24:35673
I170517 07:33:40.673690 68389 server/server.go:848  [n2] done ensuring all necessary migrations have run
I170517 07:33:40.673816 68389 server/server.go:850  [n2] serving sql connections
W170517 07:33:40.806444 68389 server/status/runtime.go:111  Could not parse build timestamp: parsing time "" as "2006/01/02 15:04:05": cannot parse "" as "2006"
W170517 07:33:40.808167 68389 gossip/gossip.go:1196  [n?] no incoming or outgoing connections
I170517 07:33:40.812128 68389 server/config.go:436  1 storage engine initialized
I170517 07:33:40.815347 68742 gossip/client.go:131  [n?] started gossip client to 127.87.51.24:41559
I170517 07:33:40.825506 68946 gossip/server.go:234  [n1] received initial cluster-verification connection from {tcp 127.87.51.24:40517}
I170517 07:33:40.827707 68389 server/node.go:454  [n?] store [n0,s0] not bootstrapped
I170517 07:33:40.828803 68389 storage/stores.go:296  [n?] read 0 node addresses from persistent storage
I170517 07:33:40.828929 68389 server/node.go:608  [n?] connecting to gossip network to verify cluster ID...
I170517 07:33:40.833641 68389 server/node.go:633  [n?] node connected via gossip and verified as part of cluster "403e3e46-9202-4288-841d-fbf0bbff3d4c"
I170517 07:33:40.836023 68926 storage/stores.go:312  [n?] wrote 1 node addresses to persistent storage
I170517 07:33:40.837883 68926 storage/stores.go:312  [n?] wrote 2 node addresses to persistent storage
I170517 07:33:40.840568 68366 sql/event_log.go:101  [n2] Event: "node_join", target: 2, info: {Descriptor:{NodeID:2 Address:{NetworkField:tcp AddressField:127.87.51.24:35673} Attrs: Locality:} ClusterID:403e3e46-9202-4288-841d-fbf0bbff3d4c StartedAt:1495006420505361613 LastUp:1495006420505361613}
I170517 07:33:40.874950 68389 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
I170517 07:33:40.880847 68389 server/node.go:344  [n?] new node allocated ID 3
I170517 07:33:40.881117 68389 gossip/gossip.go:297  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.87.51.24:40517" > attrs:<> locality:<>
I170517 07:33:40.881939 68389 server/node.go:405  [n3] node=3: started with [[]=] engine(s) and attributes []
I170517 07:33:40.884190 68389 sql/executor.go:349  [n3] creating distSQLPlanner with address {tcp 127.87.51.24:40517}
I170517 07:33:40.889097 68971 storage/stores.go:312  [n1] wrote 2 node addresses to persistent storage
I170517 07:33:40.891957 68972 storage/stores.go:312  [n2] wrote 2 node addresses to persistent storage
I170517 07:33:40.975830 68389 server/server.go:736  [n3] starting http server at 127.87.51.24:41147
I170517 07:33:40.980892 68389 server/server.go:737  [n3] starting grpc/postgres server at 127.87.51.24:40517
I170517 07:33:40.981077 68389 server/server.go:738  [n3] advertising CockroachDB node at 127.87.51.24:40517
I170517 07:33:40.995609 68389 server/server.go:848  [n3] done ensuring all necessary migrations have run
I170517 07:33:40.995733 68389 server/server.go:850  [n3] serving sql connections
I170517 07:33:41.025350 68980 server/node.go:589  [n3] bootstrapped store [n3,s3]
I170517 07:33:41.085791 69141 storage/replica_raftstorage.go:416  [n1,s1,r1/1:/M{in-ax}] generated preemptive snapshot 9f400e8d at index 61
I170517 07:33:41.103437 68974 sql/event_log.go:101  [n3] Event: "node_join", target: 3, info: {Descriptor:{NodeID:3 Address:{NetworkField:tcp AddressField:127.87.51.24:40517} Attrs: Locality:} ClusterID:403e3e46-9202-4288-841d-fbf0bbff3d4c StartedAt:1495006420881713470 LastUp:1495006420881713470}
I170517 07:33:41.126421 69141 storage/store.go:3345  [n1,s1,r1/1:/M{in-ax}] streamed snapshot to (n2,s2):?: kv pairs: 720, log entries: 3, rate-limit: 2.0 MiB/sec, 14ms
I170517 07:33:41.128076 69158 storage/replica_raftstorage.go:597  [n2,s2,r1/?:{-}] applying preemptive snapshot at index 61 (id=9f400e8d, encoded size=125222, 1 rocksdb batches, 3 log entries)
I170517 07:33:41.130128 69158 storage/replica_raftstorage.go:605  [n2,s2,r1/?:/M{in-ax}] applied preemptive snapshot in 2ms [clear=0ms batch=0ms entries=0ms commit=1ms]
I170517 07:33:41.133802 69141 storage/replica_command.go:3538  [n1,s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n2,s2):2): read existing descriptor r1:/M{in-ax} [(n1,s1):1, next=2]
I170517 07:33:41.269665 69173 storage/replica.go:2590  [n1,s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n2,s2):2: [(n1,s1):1 (n2,s2):2]
I170517 07:33:41.274673 69141 storage/replica_raftstorage.go:416  [n1,s1,r1/1:/M{in-ax}] generated preemptive snapshot a8e09998 at index 67
I170517 07:33:41.317283 69141 storage/store.go:3345  [n1,s1,r1/1:/M{in-ax}] streamed snapshot to (n3,s3):?: kv pairs: 979, log entries: 9, rate-limit: 2.0 MiB/sec, 40ms
I170517 07:33:41.319052 69186 storage/replica_raftstorage.go:597  [n3,s3,r1/?:{-}] applying preemptive snapshot at index 67 (id=a8e09998, encoded size=185384, 1 rocksdb batches, 9 log entries)
I170517 07:33:41.326226 69186 storage/replica_raftstorage.go:605  [n3,s3,r1/?:/M{in-ax}] applied preemptive snapshot in 6ms [clear=0ms batch=0ms entries=1ms commit=4ms]
I170517 07:33:41.337966 69141 storage/replica_command.go:3538  [n1,s1,r1/1:/M{in-ax}] change replicas (ADD_REPLICA (n3,s3):3): read existing descriptor r1:/M{in-ax} [(n1,s1):1, (n2,s2):2, next=3]
I170517 07:33:41.349765 69202 storage/raft_transport.go:436  [n2] raft transport stream to node 1 established
I170517 07:33:41.441856 69206 storage/replica.go:2590  [n1,s1,r1/1:/M{in-ax}] proposing ADD_REPLICA (n3,s3):3: [(n1,s1):1 (n2,s2):2 (n3,s3):3]
I170517 07:33:41.481467 69252 storage/raft_transport.go:436  [n3] raft transport stream to node 1 established
I170517 07:33:41.488493 68389 storage/gossip_test.go:181  stopping server 0
I170517 07:33:41.491171 68344 kv/transport_race.go:71  transport race promotion: ran 36 iterations on up to 135 requests
W170517 07:33:41.494690 69202 storage/raft_transport.go:442  [n2] raft transport stream to node 1 failed: EOF
I170517 07:33:41.501088 68669 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.87.51.24:41559->127.0.0.1:57370: use of closed network connection
I170517 07:33:41.502047 68666 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
I170517 07:33:41.503017 68724 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.87.51.24:41559: getsockopt: connection refused"; Reconnecting to {127.87.51.24:41559 <nil>}
I170517 07:33:41.505091 68848 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.87.51.24:41559->127.0.0.1:57380: use of closed network connection
I170517 07:33:41.505812 68353 vendor/google.golang.org/grpc/transport/http2_server.go:323  transport: http2Server.HandleStreams failed to read frame: read tcp 127.87.51.24:41559->127.0.0.1:57361: use of closed network connection
I170517 07:33:41.526304 68747 vendor/google.golang.org/grpc/transport/http2_client.go:1231  transport: http2Client.notifyError got notified that the client transport was broken EOF.
W170517 07:33:41.527922 69252 storage/raft_transport.go:442  [n3] raft transport stream to node 1 failed: rpc error: code = Internal desc = transport is closing
I170517 07:33:41.528613 68744 vendor/google.golang.org/grpc/clientconn.go:806  grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.87.51.24:41559: getsockopt: connection refused"; Reconnecting to {127.87.51.24:41559 <nil>}
W170517 07:33:41.546088 68389 server/status/runtime.go:111  Could not parse build timestamp: parsing time "" as "2006/01/02 15:04:05": cannot parse "" as "2006"
W170517 07:33:41.561127 68389 gossip/gossip.go:1196  [n?] no incoming or outgoing connections
I170517 07:33:41.575055 69272 gossip/client.go:131  [n?] started gossip client to 127.87.51.24:35673
I170517 07:33:41.575526 68389 server/config.go:436  1 storage engine initialized
I170517 07:33:41.576766 68389 server/node.go:454  [n?] store [n0,s0] not bootstrapped
I170517 07:33:41.576876 68389 storage/stores.go:296  [n?] read 0 node addresses from persistent storage
I170517 07:33:41.576969 68389 server/node.go:608  [n?] connecting to gossip network to verify cluster ID...
I170517 07:33:41.581665 69243 gossip/server.go:234  [n2] received initial cluster-verification connection from {tcp 127.87.51.24:41559}
I170517 07:33:41.587141 68389 server/node.go:633  [n?] node connected via gossip and verified as part of cluster "403e3e46-9202-4288-841d-fbf0bbff3d4c"
I170517 07:33:41.588366 69248 storage/stores.go:312  [n?] wrote 1 node addresses to persistent storage
I170517 07:33:41.593982 69248 storage/stores.go:312  [n?] wrote 2 node addresses to persistent storage
I170517 07:33:41.673368 68389 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
I170517 07:33:41.689304 68389 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
I170517 07:33:41.734767 68389 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
I170517 07:33:41.782189 68389 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
W170517 07:33:41.812863 69295 gossip/client.go:125  [n3] failed to start gossip client to 127.87.51.24:41559: rpc error: code = Unavailable desc = grpc: the connection is unavailable
I170517 07:33:41.858402 68389 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
I170517 07:33:41.910805 68389 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
I170517 07:33:41.972125 68389 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
I170517 07:33:42.036759 68389 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
I170517 07:33:42.104043 68389 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
I170517 07:33:42.166265 68389 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
I170517 07:33:42.235065 68389 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
I170517 07:33:42.290440 68389 kv/dist_sender.go:367  [n?] unable to determine this node's attributes for replica selection; node is most likely bootstrapping
W170517 07:33:42.461301 69551 gossip/client.go:125  [n2] failed to start gossip client to 127.87.51.24:41559: rpc error: code = Unavailable desc = grpc: the connection is unavailable
I170517 07:33:42.611670 69556 storage/raft_transport.go:436  [n3] raft transport stream to node 2 established
I170517 07:33:42.616324 69586 storage/raft_transport.go:436  [n2] raft transport stream to node 3 established
I170517 07:33:42.815677 69618 gossip/client.go:131  [n3] started gossip client to 127.87.51.24:35673
I170517 07:33:43.462536 69561 gossip/client.go:131  [n2] started gossip client to 127.87.51.24:40517
I170517 07:33:43.732794 69650 storage/raft_transport.go:436  [n2] raft transport stream to node 1 established
I170517 07:33:43.752339 69618 gossip/client.go:136  [n3] closing client to node 2 (127.87.51.24:35673): stopping outgoing client to node 2 (127.87.51.24:35673); already have incoming
I170517 07:33:43.752864 68923 gossip/gossip.go:1210  [n3] node has connected to cluster via gossip
I170517 07:33:43.753424 68923 storage/stores.go:312  [n3] wrote 2 node addresses to persistent storage
E170517 07:33:43.762943 69575 util/log/crash_reporting.go:153  [n3,s3,r1/3:/M{in-ax}] Reported as error 264820ef0202419da40bc967f39b77c0
F170517 07:33:43.763059 69575 storage/replica.go:1360  [n3,s3,r1/3:/M{in-ax}] on-disk and in-memory state diverged:
[RaftAppliedIndex: 75 != 74]
goroutine 69575 [running]:
github.com/cockroachdb/cockroach/pkg/util/log.getStacks(0x3ef7601, 0x2d7b5c4f, 0x3535700, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:858 +0xc4
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).outputLogEntry(0x3536ec0, 0xc400000004, 0x2ded4d5, 0x12, 0x550, 0xc4208f99e0, 0x59)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:726 +0x6e7
github.com/cockroachdb/cockroach/pkg/util/log.addStructured(0x7f4ce5286040, 0xc421c396b0, 0x4, 0x2, 0x2279083, 0x28, 0xc420e8a998, 0x1, 0x1)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/structured.go:158 +0x5ea
github.com/cockroachdb/cockroach/pkg/util/log.logDepth(0x7f4ce5286040, 0xc421c396b0, 0x1, 0x4, 0x2279083, 0x28, 0xc420e8a998, 0x1, 0x1)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:68 +0x9a
github.com/cockroachdb/cockroach/pkg/util/log.Fatalf(0x7f4ce5286040, 0xc421c396b0, 0x2279083, 0x28, 0xc420e8a998, 0x1, 0x1)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:162 +0x90
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).assertStateLocked(0xc420507880, 0x7f4ce5286040, 0xc421c396b0, 0x30cf540, 0xc42027edc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1360 +0x4a1
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).assertState(0xc420507880, 0x7f4ce5286040, 0xc421c396b0, 0x30cf540, 0xc42027edc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1346 +0xae
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleEvalResult(0xc420507880, 0x7f4ce5286040, 0xc421c396b0, 0xc42165a000, 0x3ef7888, 0x0, 0x0, 0xc42020f150, 0x1, 0x1, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_proposal.go:802 +0x17c
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc420507880, 0x7f4ce5286040, 0xc421c396b0, 0x0, 0x0, 0x14bf546b90031c35, 0x0, 0x100000001, 0x1, 0x14bf546b90031c35, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2488 +0x350
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc420507880, 0x7f4ce5286040, 0xc421c396b0, 0x14bf546b917ba52c, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2237 +0x3bf
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc420507880, 0x7f4ce5286040, 0xc421c396b0, 0x14bf546b917ba52c, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2091 +0xa9
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420507880, 0x7f4ce5286040, 0xc421c396b0, 0x14bf546b917ba52c, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1425 +0x28e
github.com/cockroachdb/cockroach/pkg/storage.(*pendingLeaseRequest).requestLeaseAsync.func1(0x7f4ce5286040, 0xc4201e3620)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_range_lease.go:213 +0x65e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc421a4c280, 0x7f4ce651c918, 0xc420016e40, 0x2ded282, 0x1e, 0xf3, 0x0, 0x0, 0xc4205c8510)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:294 +0x101
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:295 +0x173

goroutine 1 [chan receive]:
testing.(*T).Run(0xc420076a90, 0x226280b, 0x1d, 0x2310bd0, 0xc420683901)
	/usr/local/go/src/testing/testing.go:698 +0x582
testing.runTests.func1(0xc420076a90)
	/usr/local/go/src/testing/testing.go:882 +0xab
testing.tRunner(0xc420076a90, 0xc420683ad0)
	/usr/local/go/src/testing/testing.go:657 +0x108
testing.runTests(0xc4201a5640, 0x30a8a00, 0x1a6, 0x1a6, 0xc420683b50)
	/usr/local/go/src/testing/testing.go:888 +0x4e1
testing.(*M).Run(0xc420683f28, 0x0)
	/usr/local/go/src/testing/testing.go:822 +0x1c4
github.com/cockroachdb/cockroach/pkg/storage_test.TestMain(0xc420683f28)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/main_test.go:59 +0x308
main.main()
	github.com/cockroachdb/cockroach/pkg/storage/_test/_testmain.go:894 +0x210

goroutine 17 [syscall, 1 minutes, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1

goroutine 5 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).flushDaemon(0x3536ec0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1024 +0x85
created by github.com/cockroachdb/cockroach/pkg/util/log.init.1
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:566 +0x13a

goroutine 7 [syscall, 1 minutes]:
os/signal.signal_recv(0x6cfed1)
	/usr/local/go/src/runtime/sigqueue.go:116 +0x104
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x30
created by os/signal.init.1
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x4f

goroutine 68766 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4201967e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337370, 0xc42096e780, 0xc420337360)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68755 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).startSampleEnvironment.func1(0x7f4ce5286040, 0xc420de6d20)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:945 +0x1ce
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337220, 0xc42096e780, 0xc4201d1c00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69002 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000006a)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38360)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380a40, 0xc421a4c280, 0xc420214d00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68897 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func2(0x7f4ce5286040, 0xc4223d3e60)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:113 +0x48f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420294bf0, 0xc421a4c280, 0xc420352960)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68915 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x7f4ce5286040, 0xc420700d50)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:102 +0x8e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420295310, 0xc421a4c280, 0xc420352ee0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69285 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc421929a00, 0xc4201db090, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:143 +0x70a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc420aa58c0, 0xc4201db090, 0x5, 0x5, 0xc421b0ba48, 0x0, 0xc4211fc9c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:333 +0x7f
io.ReadAtLeast(0x30b4640, 0xc420aa58c0, 0xc4201db090, 0x5, 0x5, 0x5, 0xc421b0bb10, 0x67730d, 0xc420c92ea0)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30b4640, 0xc420aa58c0, 0xc4201db090, 0x5, 0x5, 0x69fb14, 0xc421b0bae0, 0x6cbed0)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4201db080, 0x7fffffff, 0x1d0d7c0, 0x7f4cd2544408, 0xc42045c840, 0x0, 0x0, 0xc421b0bbd0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:243 +0xa0
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4201db080, 0x30c6f00, 0x3ef7888, 0xc420aa58c0, 0x30bc000, 0x3ef7888, 0x2166560, 0xc42100e960, 0x7fffffff, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:339 +0x61
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc4218e2dc0, 0x2166560, 0xc42100e960, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:386 +0x20e
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc4202fad90, 0x7f4ce5286040, 0xc42033ccf0, 0xc4219b2900)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:194 +0x86
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0x30cf6c0, 0xc4202fad90, 0xc420956680, 0x7f4ce5286040, 0xc42033ccf0, 0xc4219b2900, 0xc421955f68, 0x69c512)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:313 +0x43
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2(0x7f4ce5286040, 0xc42033ccf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:321 +0xe6
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202faed0, 0xc42056f5e0, 0xc421929a80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68712 [chan receive]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x7f4ce5286040, 0xc42048a180)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:386 +0x94
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420260110, 0xc42096e780, 0xc4203ac500)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68878 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c3f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203375b0, 0xc42096e780, 0xc4203375a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68778 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000004f)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421485020)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202615f0, 0xc42096e780, 0xc4203ad040)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68709 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x7f4ce5286040, 0xc420b3ff80)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:547 +0xd2
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420260000, 0xc42096e780, 0xc421bec100)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68804 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000068)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64ed0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261820, 0xc42096e780, 0xc4203ad440)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68810 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000006e)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e65020)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261890, 0xc42096e780, 0xc4203ad520)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69532 [IO wait]:
net.runtime_pollWait(0x7f4ce52cd678, 0x72, 0x6fb68a)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc421b85028, 0x72, 0x30adc98, 0xffffffffffffffff)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc421b85028, 0xc421006000, 0x8000)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).Read(0xc421b84fc0, 0xc421006000, 0x8000, 0x8000, 0x0, 0x30b8100, 0x30adc98)
	/usr/local/go/src/net/fd_unix.go:250 +0x22b
net.(*conn).Read(0xc420e78090, 0xc421006000, 0x8000, 0x8000, 0x11f93c3, 0xc4204a44d0, 0x7f4ce5286040)
	/usr/local/go/src/net/net.go:181 +0x97
net.(*TCPConn).Read(0xc420e78090, 0xc421006000, 0x8000, 0x8000, 0xc42027c000, 0x307350a6, 0xc4202f4620)
	<autogenerated>:72 +0x73
bufio.(*Reader).Read(0xc421cf4a80, 0xc4202c6e38, 0x9, 0x9, 0x6a01c5, 0xc420239800, 0xc420020600)
	/usr/local/go/src/bufio/bufio.go:213 +0x7e7
io.ReadAtLeast(0x30af940, 0xc421cf4a80, 0xc4202c6e38, 0x9, 0x9, 0x9, 0xc420453a00, 0xc421cf4840, 0xc420defe90)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30af940, 0xc421cf4a80, 0xc4202c6e38, 0x9, 0x9, 0x2, 0x2, 0x1)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4202c6e38, 0x9, 0x9, 0x30af940, 0xc421cf4a80, 0x0, 0x0, 0x1d0b200, 0x7f4cd7d1b708)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4202c6e00, 0x6cc760, 0x6ca130, 0xc421cf48a0, 0x2)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x118
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc420b253e0, 0x98c384, 0x98c301, 0xc420deffa8, 0x98c30e)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc420bbaa80)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1028 +0x7e
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:251 +0x1294

goroutine 68805 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000069)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64f00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261830, 0xc42096e780, 0xc4203ad460)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68991 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc422908870)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203314f0, 0xc421a4c280, 0xc4203314e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69267 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4202533c0, 0x2311fe0, 0xc4202c7b20, 0x30c6900, 0xc4202533c0)
	<autogenerated>:9 +0x9f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve(0xc4202c7b20, 0x30c6900, 0xc4202533c0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:396 +0x1f8
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func7(0x7f4ce5286040, 0xc421a38f90)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:558 +0x8d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020f140, 0xc42056f5e0, 0xc420253560)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69053 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000005d)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e1860)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340940, 0xc421a4c280, 0xc4201e8e00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69315 [IO wait]:
net.runtime_pollWait(0x7f4ce653c2c8, 0x72, 0x6fb68a)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc4227de378, 0x72, 0x30adc98, 0xffffffffffffffff)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc4227de378, 0xc421cae000, 0x8000)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).Read(0xc4227de310, 0xc421cae000, 0x8000, 0x8000, 0x0, 0x30b8100, 0x30adc98)
	/usr/local/go/src/net/fd_unix.go:250 +0x22b
net.(*conn).Read(0xc42283a338, 0xc421cae000, 0x8000, 0x8000, 0x69cbeb, 0xc420020600, 0x0)
	/usr/local/go/src/net/net.go:181 +0x97
net.(*TCPConn).Read(0xc42283a338, 0xc421cae000, 0x8000, 0x8000, 0x67ee91, 0x69fb14, 0xc4208f3b00)
	<autogenerated>:72 +0x73
bufio.(*Reader).Read(0xc4208f9140, 0xc420af4818, 0x9, 0x9, 0x0, 0x0, 0xc4208f3be0)
	/usr/local/go/src/bufio/bufio.go:213 +0x7e7
io.ReadAtLeast(0x30af940, 0xc4208f9140, 0xc420af4818, 0x9, 0x9, 0x9, 0xc420efe9c0, 0xffffffff, 0xc42024acd0)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30af940, 0xc4208f9140, 0xc420af4818, 0x9, 0x9, 0xd70eb8, 0xc421c96bb0, 0x0)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420af4818, 0x9, 0x9, 0x30af940, 0xc4208f9140, 0x0, 0xc400000000, 0xc420d24240, 0x30b0040)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420af47e0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x118
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc42048a7e0, 0xc420b247e0, 0xc420b24700, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc421c96a80)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1043 +0x15b
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:251 +0x1294

goroutine 69489 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4228e8440, 0xc4202929b0, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:143 +0x70a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42137a5a0, 0xc4202929b0, 0x5, 0x5, 0x1d0d7c0, 0x7f4cbdaa0188, 0xc420526498)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:333 +0x7f
io.ReadAtLeast(0x30b4640, 0xc42137a5a0, 0xc4202929b0, 0x5, 0x5, 0x5, 0xc420526498, 0x6d104c, 0x6ef873)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30b4640, 0xc42137a5a0, 0xc4202929b0, 0x5, 0x5, 0xc420526498, 0x4, 0x6d0f42)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4202929a0, 0x7fffffff, 0x68, 0x68, 0xc4217fdd18, 0x69cbeb, 0x683e7d, 0xc41ffeb657)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:243 +0xa0
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4202929a0, 0x30c6f00, 0x3ef7888, 0xc42137a5a0, 0x30bc000, 0x3ef7888, 0x216efa0, 0xc420293500, 0x7fffffff, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:339 +0x61
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc420efc0a0, 0x216efa0, 0xc420293500, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:613 +0x1d5
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc4201a6620, 0x7f4ce5286040, 0xc4228e6ae0, 0xc4223d6470)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:397 +0x86
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x30d0740, 0xc4201a6620, 0xc4217b9880, 0x7f4ce5286040, 0xc4228e6ae0, 0x18, 0xc420e2b778)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:300 +0x61
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1(0x7f4ce5286040, 0xc4228e6ae0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:323 +0x75
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201a6640, 0xc42096e780, 0xc4228e6ab0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68923 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x7f4ce5286040, 0xc420701590)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1093 +0x415
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420295590, 0xc421a4c280, 0xc420295580)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69052 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000058)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e16b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340930, 0xc421a4c280, 0xc4201e8de0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68824 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000007c)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e65350)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202619c0, 0xc42096e780, 0xc4203ad780)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68788 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000056)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421485140)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261690, 0xc42096e780, 0xc4203ad1e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68745 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420fe0120, 0xc420396400, 0xc420352120, 0x12, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:361 +0xb5b
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.2.1(0x7f4ce5286040, 0xc420701440)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:297 +0x86
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203307f0, 0xc421a4c280, 0xc420701320)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69644 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4211cfcc0, 0xc4203d6930, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:143 +0x70a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc4215839e0, 0xc4203d6930, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:333 +0x7f
io.ReadAtLeast(0x30b4640, 0xc4215839e0, 0xc4203d6930, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30b4640, 0xc4215839e0, 0xc4203d6930, 0x5, 0x5, 0x0, 0x683bf2, 0x0)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4203d6920, 0x7fffffff, 0xc420e30660, 0x5, 0x5, 0xc420e30750, 0x5, 0x5)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:243 +0xa0
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4203d6920, 0x30c6f00, 0x3ef7888, 0xc4215839e0, 0x30bc000, 0x3ef7888, 0x216f080, 0xc42033de90, 0x7fffffff, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:339 +0x61
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc4218e2840, 0x216f080, 0xc42033de90, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:386 +0x20e
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc420260a80, 0xc420e306a8, 0x7f4ce5286040, 0xc42033de80)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:329 +0x86
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x30d06e0, 0xc420260a80, 0xc4202a16c0, 0xc4217b9880, 0x7f4ce5286040, 0xc42033de60, 0x18, 0xc420e30778)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:466 +0x5a
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1(0x7f4ce5286040, 0xc42033de60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:483 +0x7e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420260af0, 0xc42096e780, 0xc42033dd70)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68791 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000005a)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421485200)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202616c0, 0xc42096e780, 0xc4203ad240)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68775 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000004c)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421484f90)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202615b0, 0xc42096e780, 0xc4203acfc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68992 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4229088d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331580, 0xc421a4c280, 0xc420331570)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69061 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000061)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e18f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203409f0, 0xc421a4c280, 0xc4201e8fa0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69505 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc42218c900, 0x23120e8, 0xc421915598, 0x30cea60)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:252 +0x220
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x7f4ce5286040, 0xc420b3e780, 0x0, 0x0, 0x30c6f00, 0x3ef7888, 0x0, 0x0, 0x30bc000, 0x3ef7888, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:63 +0xc9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7f4ce5286040, 0xc420b3e780, 0x2269a28, 0x21, 0x221faa0, 0xc420650868, 0x21926e0, 0xc421bf4c60, 0xc42054d200, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:265 +0xdf9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x7f4ce5286040, 0xc420b3e780, 0x2269a28, 0x21, 0x221faa0, 0xc420650868, 0x21926e0, 0xc421bf4c60, 0xc42054d200, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:146 +0x217
github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4214a40c0, 0x7f4ce5286040, 0xc420b3e780, 0xc420650868, 0x0, 0x0, 0x0, 0x1, 0xc420dfe700, 0xc421d26b20)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1854 +0x107
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc421ee2050, 0xc420650840, 0x7f4ce5286040, 0xc420b3e780, 0x149f55b, 0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:261 +0x253
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421ee2050, 0xc420650840, 0x7f4ce5286040, 0xc420b3e780, 0x30d3200, 0x3ef7888, 0xc421ce9980)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:270 +0xdd
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:283 +0x236

goroutine 68744 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc42139fd40)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:859 +0x7cf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc42139fd40)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:604 +0x267
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:605 +0xa4d

goroutine 68924 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open.func2(0xc42027edc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:430 +0x64
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:431 +0x6d1

goroutine 68790 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000059)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4214851d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202616b0, 0xc42096e780, 0xc4203ad220)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68906 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc420e65f20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ae790, 0xc421a4c280, 0xc4202be4e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69309 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc4211fcea0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:859 +0x7cf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc4211fcea0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:604 +0x267
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:605 +0xa4d

goroutine 68706 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x7f4ce5286040, 0xc42048a0c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:102 +0x8e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420193c40, 0xc42096e780, 0xc420198720)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69525 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc420934120, 0x23120e8, 0xc42067d598, 0x30cea60)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:252 +0x220
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x7f4ce5286040, 0xc420b24900, 0x0, 0x0, 0x30c6f00, 0x3ef7888, 0x0, 0x0, 0x30bc000, 0x3ef7888, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:63 +0xc9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7f4ce5286040, 0xc420b24900, 0x2269a28, 0x21, 0x221faa0, 0xc4204935e8, 0x21926e0, 0xc42093d200, 0xc4219b2d80, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:265 +0xdf9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x7f4ce5286040, 0xc420b24900, 0x2269a28, 0x21, 0x221faa0, 0xc4204935e8, 0x21926e0, 0xc42093d200, 0xc4219b2d80, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:146 +0x217
github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc420e78058, 0x7f4ce5286040, 0xc420b24900, 0xc4204935e8, 0x0, 0x0, 0x0, 0x1, 0x3537000, 0xc4212a3b60)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1854 +0x107
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc420230780, 0xc4204935c0, 0x7f4ce5286040, 0xc420b24900, 0x180001, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:261 +0x253
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420230780, 0xc4204935c0, 0x7f4ce5286040, 0xc420b24900, 0x30d3200, 0x3ef7888, 0xc42093d0e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:270 +0xdd
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:283 +0x236

goroutine 69062 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000062)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc42050f590)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340a20, 0xc421a4c280, 0xc4201e9020)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69055 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000057)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e1680)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340960, 0xc421a4c280, 0xc4201e8e60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69589 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc42007a600, 0x30d0740, 0xc4201a6fd0, 0x30b9d80, 0xc42007a600)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:329 +0x32d
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x22125a0, 0xc42007a600, 0x30cd8c0, 0xc420efc460, 0xc420e30dd8, 0xc42137ac60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:378 +0xd6
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc4200a41c0, 0x30ceac0, 0xc420ce1180, 0xc42137ac60, 0xc421485470, 0x30994a0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:842 +0xe22
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4200a41c0, 0x30ceac0, 0xc420ce1180, 0xc42137ac60, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:936 +0x11c2
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420351940, 0xc4200a41c0, 0x30ceac0, 0xc420ce1180, 0xc42137ac60)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:497 +0xb7
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:498 +0xb9

goroutine 68907 [semacquire]:
sync.runtime_Semacquire(0xc420507984)
	/usr/local/go/src/runtime/sema.go:47 +0x34
sync.(*RWMutex).RLock(0xc420507978)
	/usr/local/go/src/sync/rwmutex.go:43 +0x95
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Desc(0xc420507880, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1154 +0x54
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).requiresSplit(0xc42095ab00, 0xc420df9800, 0x18, 0x20, 0xc420507880, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:381 +0x85
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).MaybeAdd(0xc42095ab00, 0xc420507880, 0x14bf546beaa329b7, 0xc400000000)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:322 +0x880
github.com/cockroachdb/cockroach/pkg/storage.(*gcQueue).MaybeAdd(0xc42017e0b8, 0xc420507880, 0x14bf546beaa329b7, 0x0)
	<autogenerated>:198 +0x7c
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc421587500, 0x7f4ce5286040, 0xc420e65f50, 0xed0adf3d7, 0x2246c09d, 0x3535700, 0xc421d142c0, 0xc421a4c280, 0xc420507880, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:224 +0x393
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc420507880, 0x58)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:272 +0x162
github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc420e27cb0, 0xc421bb7b30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:358 +0x5ad
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1(0x7f4ce5286040, 0xc420e65f50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x3f0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ae7a0, 0xc421a4c280, 0xc4202be500)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68890 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4204f4800, 0x7f4ce5286000, 0xc42284b280, 0x1dcd6500, 0x0, 0xc4204f4848, 0x1, 0xc4202ed6d0, 0x3, 0x3, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1237 +0x1cef
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4204f4800, 0x7f4ce5286000, 0xc42284b280, 0x1, 0xc4202ed6d0, 0x3, 0x3, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:405 +0x3b4
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4204f4800, 0x7f4ce5286040, 0xc420ac7ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420957c70, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:474 +0x1a8
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4204f4800, 0x7f4ce5286040, 0xc420ac7ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420957c70, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:940 +0x4ec
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4204f4800, 0x7f4ce5286040, 0xc420ac7ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420957c70, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:810 +0xb78
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4204f4800, 0x7f4ce5286040, 0xc420ac7ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421079c70, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:624 +0x3ce
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc420c8e270, 0x7f4ce5286040, 0xc420ac7e00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421079ad0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:478 +0x490
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc420574ae0, 0x7f4ce5286000, 0xc42177cc40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421079ad0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:528 +0x228
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).send(0xc420e05ce0, 0x7f4ce5286000, 0xc42177cc40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:864 +0x59b
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x7f4ce5286000, 0xc42177cc40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:434 +0x97
github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x7f4ce5286000, 0xc42177cc40, 0xc4211af638, 0xc4228aa000, 0xc4218bea80, 0xc4211af760)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:436 +0x13b
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc420e05ce0, 0x7f4ce5286000, 0xc42177cc40, 0xc4228aa000, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:434 +0x112
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).updateLiveness.func1(0x7f4ce5286000, 0xc42177cc40, 0xc420e05ce0, 0xc4211af7d8, 0xbf41cc)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:545 +0x599
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Txn.func1(0x7f4ce5286000, 0xc42177cc40, 0xc420e05ce0, 0xc4203945ab, 0xc4211af8f8, 0x6ac6f2)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:490 +0x5f
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Exec(0xc420e05ce0, 0x7f4ce5286000, 0xc42177cc40, 0xc421010101, 0xc4203946f0, 0x28, 0xc42115c270)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:640 +0xe3
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Txn(0xc420574ae0, 0x7f4ce5286000, 0xc42177cc40, 0xc4202e4e20, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x17a
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).updateLiveness(0xc42115c1e0, 0x7f4ce5286040, 0xc420ac7590, 0xc420ac7620, 0xc420ac75c0, 0xc4211afbb0, 0x3535700, 0x1000000006d0f01)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:546 +0x18d
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).heartbeatInternal.func1(0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:333 +0x4e5
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).heartbeatInternal(0xc42115c1e0, 0x7f4ce5286040, 0xc420ac7590, 0xc420ac75c0, 0x3ff8000000000000, 0x0, 0x3fc3333333333333)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:349 +0x130
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1(0x7f4ce5286040, 0xc42038d0e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:248 +0x52e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331740, 0xc42096e780, 0xc42026a910)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69134 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc4211d6300)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1162 +0x6e6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:283 +0x1a4b

goroutine 68333 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc420de7380)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420294380, 0xc42096e780, 0xc4202f4640)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69106 [IO wait]:
net.runtime_pollWait(0x7f4ce653d108, 0x72, 0x6fb68a)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc4203ce0d8, 0x72, 0x30adc98, 0xffffffffffffffff)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc4203ce0d8, 0xc4215f4000, 0x8000)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).Read(0xc4203ce070, 0xc4215f4000, 0x8000, 0x8000, 0x0, 0x30b8100, 0x30adc98)
	/usr/local/go/src/net/fd_unix.go:250 +0x22b
net.(*conn).Read(0xc4204a4040, 0xc4215f4000, 0x8000, 0x8000, 0x69cbeb, 0xc420022c00, 0x0)
	/usr/local/go/src/net/net.go:181 +0x97
net.(*TCPConn).Read(0xc4204a4040, 0xc4215f4000, 0x8000, 0x8000, 0x0, 0xc42022f5e0, 0x10)
	<autogenerated>:72 +0x73
bufio.(*Reader).Read(0xc421a46540, 0xc4202c6738, 0x9, 0x9, 0x0, 0x0, 0xc420db0be0)
	/usr/local/go/src/bufio/bufio.go:213 +0x7e7
io.ReadAtLeast(0x30af940, 0xc421a46540, 0xc4202c6738, 0x9, 0x9, 0x9, 0xc420a72d00, 0xffffffff, 0xc420c1d7c0)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30af940, 0xc421a46540, 0xc4202c6738, 0x9, 0x9, 0xd70eb8, 0xc4227a0730, 0x0)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4202c6738, 0x9, 0x9, 0x30af940, 0xc421a46540, 0x0, 0xc400000000, 0xc42218c120, 0x30b0040)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4202c6700, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x118
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc42038cd80, 0xc4221f02a0, 0xc4221f0200, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc4227a0600)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1043 +0x15b
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:251 +0x1294

goroutine 68832 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x7f4ce5286040, 0xc420e654a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1351 +0x64e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261af0, 0xc42096e780, 0xc4203782d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68947 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1(0x7f4ce5286040, 0xc4204178f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:158 +0x74
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340770, 0xc421a4c280, 0xc4201e8a20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68761 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc420de6e40)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203372d0, 0xc42096e780, 0xc4203372c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68829 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1(0x7f4ce5286040, 0xc420e65440)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3486 +0x481
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261aa0, 0xc42096e780, 0xc420261a90)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69211 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).printStatsLoop(0xc4219cf790, 0x7f4ce5286040, 0xc4221c33e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:253 +0xfce
github.com/cockroachdb/cockroach/pkg/kv.NewTxnCoordSender.func1(0x7f4ce5286040, 0xc4221c33e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:237 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020e590, 0xc42056f5e0, 0xc42020e580)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68776 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000004e)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421484ff0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202615d0, 0xc42096e780, 0xc4203ad000)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69645 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/getsentry/raven-go.(*Client).worker(0xc42039eee0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/getsentry/raven-go/client.go:482 +0x94
created by github.com/cockroachdb/cockroach/vendor/github.com/getsentry/raven-go.(*Client).Capture.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/getsentry/raven-go/client.go:541 +0x56

goroutine 69216 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func2(0x7f4ce5286040, 0xc420b3e270)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:520 +0x95
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020f110, 0xc42056f5e0, 0xc420253520)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68868 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4201969c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337430, 0xc42096e780, 0xc420337420)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68795 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000005f)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64d50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261750, 0xc42096e780, 0xc4203ad2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68983 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).startSampleEnvironment.func1(0x7f4ce5286040, 0xc422908b70)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:945 +0x1ce
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331310, 0xc421a4c280, 0xc420346da0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69224 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc4200a4ee0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:742 +0xa23
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:161 +0xdf2

goroutine 69016 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4229089f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203316e0, 0xc421a4c280, 0xc4203316c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68725 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open.func2(0xc420702580)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:430 +0x64
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:431 +0x6d1

goroutine 68784 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000053)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4214850e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261650, 0xc42096e780, 0xc4203ad140)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69533 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc420bbaa80)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1116 +0x912
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:281 +0x1954

goroutine 68894 [chan receive]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1(0x7f4ce5286040, 0xc420700030)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:176 +0x95
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420294320, 0xc421a4c280, 0xc4203521c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69034 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4201e2f60)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203319f0, 0xc421a4c280, 0xc4203319e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69643 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func3(0x30cea60, 0xc420bba900, 0xc4215839e0, 0xc4218e2840)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:236 +0x52e
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:256 +0x1389

goroutine 69535 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func2(0xc4212adf88, 0xc421cf5320, 0xc420507880, 0x7f4ce5286040, 0xc420b25a70, 0xc421b85490, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1075 +0x72c
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc420507880, 0x7f4ce5286040, 0xc420b25a70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1125 +0x25d
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc420507880, 0x7f4ce5286040, 0xc420b25a70, 0x14bf546b892ede5f, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0xc4219ce4e0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2204 +0x1298
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc420507880, 0x7f4ce5286040, 0xc420b25a70, 0x14bf546b892ede5f, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0xc4219ce4e0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2091 +0xa9
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420507880, 0x7f4ce5286040, 0xc420b25a70, 0x14bf546b892ede5f, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0xc4219ce4e0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1425 +0x28e
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4204d6800, 0x7f4ce5286040, 0xc420b25a40, 0x14bf546b892ede5f, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0xc4219ce4e0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2555 +0x9e0
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42007a700, 0x7f4ce5286040, 0xc420b25a10, 0x0, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0xc4219ce4e0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x24b
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x7f4ce5286040, 0xc420b25a10, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:843 +0x20f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421a4c280, 0x7f4ce5286040, 0xc420b259e0, 0xc4212af6a8, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:272 +0x14f
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4200b4000, 0x7f4ce5286040, 0xc420b259e0, 0xc421b852d0, 0xc420b259e0, 0x0, 0x6ac3c4)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:854 +0x1c1
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4200b4000, 0x7f4ce5286040, 0xc420b259e0, 0xc421b852d0, 0xc4200b4000, 0x7f4ce52d52b0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:871 +0xb8
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x22123e0, 0xc4200b4000, 0x7f4ce5286040, 0xc420b257d0, 0xc421b85260, 0x0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1877 +0x35b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4200a41c0, 0x30ceac0, 0xc420ce1180, 0xc420934480, 0xc420e262d0, 0x308e2d0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:738 +0xf68
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4200a41c0, 0x30ceac0, 0xc420ce1180, 0xc420934480, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:932 +0x122b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420351940, 0xc4200a41c0, 0x30ceac0, 0xc420ce1180, 0xc420934480)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:497 +0xb7
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:498 +0xb9

goroutine 68985 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteSummaries.func1(0x7f4ce5286040, 0xc422908720)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:739 +0x3b5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331360, 0xc421a4c280, 0xc420346de0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69282 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc420fdf500)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1116 +0x912
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:281 +0x1954

goroutine 68880 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c3c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203375f0, 0xc42096e780, 0xc4203375e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69024 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4221e0e70)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331880, 0xc421a4c280, 0xc420331870)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69028 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4221e0ed0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331900, 0xc421a4c280, 0xc4203318f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69271 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x7f4ce5286040, 0xc420b3e420)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1093 +0x415
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020f290, 0xc42056f5e0, 0xc42020f280)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68814 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000072)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e65110)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202618f0, 0xc42096e780, 0xc4203ad5c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68823 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000007b)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e652f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202619b0, 0xc42096e780, 0xc4203ad740)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68801 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000065)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64e40)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202617e0, 0xc42096e780, 0xc4203ad3c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68777 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000004d)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421484fc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202615e0, 0xc42096e780, 0xc4203ad020)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68721 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000086)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420de7020)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261540, 0xc42096e780, 0xc4203acee0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68806 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000006a)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64f60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261840, 0xc42096e780, 0xc4203ad480)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68815 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000073)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e65140)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261900, 0xc42096e780, 0xc4203ad600)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68816 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000074)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e65170)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261930, 0xc42096e780, 0xc4203ad620)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68793 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000005d)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421485230)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261700, 0xc42096e780, 0xc4203ad280)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68896 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*LeaseManager).RefreshLeases.func1(0x7f4ce5286040, 0xc4223d3e30)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/lease.go:1240 +0xf43
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420294be0, 0xc421a4c280, 0xc4214859e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69481 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc422886600)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1116 +0x912
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:281 +0x1954

goroutine 69279 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip(0xc4204f4700, 0x30cf720, 0xc4202886b0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:200 +0x772
github.com/cockroachdb/cockroach/pkg/gossip._Gossip_Gossip_Handler(0x220b100, 0xc4204f4700, 0x30cd8c0, 0xc4217b6820, 0xc4208275d8, 0xc42077e480)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:211 +0xd6
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc4203355e0, 0x30ceac0, 0xc42167a0e0, 0xc42077e480, 0xc4214a7920, 0x3091180, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:842 +0xe22
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4203355e0, 0x30ceac0, 0xc42167a0e0, 0xc42077e480, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:936 +0x11c2
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42020f5b0, 0xc4203355e0, 0x30ceac0, 0xc42167a0e0, 0xc42077e480)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:497 +0xb7
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:498 +0xb9

goroutine 68787 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000058)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4214851a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261680, 0xc42096e780, 0xc4203ad1c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68899 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc420e65da0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ae720, 0xc421a4c280, 0xc4202be380)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69045 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000052)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420417d70)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203408c0, 0xc421a4c280, 0xc4201e8ce0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69049 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000055)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420417e00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340900, 0xc421a4c280, 0xc4201e8d60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68715 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1(0x7f4ce5286040, 0xc420de6e70)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:158 +0x74
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202614e0, 0xc42096e780, 0xc4203ace00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68780 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000004b)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421484f60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261610, 0xc42096e780, 0xc4203ad080)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69284 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func3(0x30cea60, 0xc420fdf500, 0xc420aa58c0, 0xc4218e2dc0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:236 +0x52e
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:256 +0x1389

goroutine 68800 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000064)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64e10)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202617d0, 0xc42096e780, 0xc4203ad380)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69044 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000004f)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4203ba690)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203408b0, 0xc421a4c280, 0xc4201e8ca0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69217 [IO wait]:
net.runtime_pollWait(0x7f4ce653c448, 0x72, 0x30adc98)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc4216eaa08, 0x72, 0x30adc98, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc4216eaa08, 0xffffffffffffffff, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).accept(0xc4216ea9a0, 0x0, 0x30b54c0, 0xc420360f00)
	/usr/local/go/src/net/fd_unix.go:430 +0x2c0
net.(*TCPListener).accept(0xc4204a43a0, 0xc421a38f50, 0xc421a38f30, 0x69c512)
	/usr/local/go/src/net/tcpsock_posix.go:136 +0x51
net.(*TCPListener).Accept(0xc4204a43a0, 0xc421a38f00, 0x2068c40, 0x308d090, 0x2157f40)
	/usr/local/go/src/net/tcpsock.go:228 +0x50
net/http.(*Server).Serve(0xc420e29970, 0x30c5e40, 0xc4204a43a0, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2643 +0x2a8
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func5(0x7f4ce5286040, 0xc421a38ea0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:543 +0x8f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020f120, 0xc42056f5e0, 0xc420253540)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69201 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420cea340)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:859 +0x7cf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420cea340)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:604 +0x267
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:605 +0xa4d

goroutine 68993 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc422908990)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203315b0, 0xc421a4c280, 0xc4203315a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68758 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42048bcb0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337270, 0xc42096e780, 0xc420337260)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68817 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000075)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e651a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261940, 0xc42096e780, 0xc4203ad640)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69048 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000053)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4203ba720)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203408f0, 0xc421a4c280, 0xc4201e8d40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68884 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc421070450)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337670, 0xc42096e780, 0xc420337660)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69059 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000005f)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420417e90)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203409d0, 0xc421a4c280, 0xc4201e8f20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68811 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000006f)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e65050)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202618a0, 0xc42096e780, 0xc4203ad560)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68867 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc420196a80)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337410, 0xc42096e780, 0xc420337400)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68922 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x7f4ce5286040, 0xc420701020)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1055 +0x65b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420295560, 0xc421a4c280, 0xc420295550)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69080 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000076)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38600)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380bb0, 0xc421a4c280, 0xc420214f80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68953 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000045)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e15f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203407f0, 0xc421a4c280, 0xc4201e8b00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69588 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4228e8680, 0xc420292e30, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:143 +0x70a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42137ab40, 0xc420292e30, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:333 +0x7f
io.ReadAtLeast(0x30b4640, 0xc42137ab40, 0xc420292e30, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30b4640, 0xc42137ab40, 0xc420292e30, 0x5, 0x5, 0x0, 0x683bf2, 0x0)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc420292e20, 0x7fffffff, 0x6ff883, 0x308d1a0, 0x11, 0x20e3a420, 0x96ccc9, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:243 +0xa0
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc420292e20, 0x30c6f00, 0x3ef7888, 0xc42137ab40, 0x30bc000, 0x3ef7888, 0x216f080, 0xc4228e7110, 0x7fffffff, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:339 +0x61
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc420e76420, 0x216f080, 0xc4228e7110, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:386 +0x20e
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc4201a6f10, 0xc420e3a6a8, 0x7f4ce5286040, 0xc4228e7100)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:329 +0x86
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x30d06e0, 0xc4201a6f10, 0xc4228e8500, 0xc4217b9880, 0x7f4ce5286040, 0xc4228e70e0, 0x18, 0xc420e3a778)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:466 +0x5a
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1(0x7f4ce5286040, 0xc4228e70e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:483 +0x7e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201a6f60, 0xc42096e780, 0xc4228e70b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68768 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4210703f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203373b0, 0xc42096e780, 0xc4203373a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68767 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc420196930)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337390, 0xc42096e780, 0xc420337380)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68794 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000005e)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420de7140)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261740, 0xc42096e780, 0xc4203ad2a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69095 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x7f4ce5286040, 0xc420e65c80)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1351 +0x64e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380ce0, 0xc421a4c280, 0xc4202bd6d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68763 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c0f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337310, 0xc42096e780, 0xc420337300)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69046 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000051)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e1620)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203408d0, 0xc421a4c280, 0xc4201e8d00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68335 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc420de73e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202943a0, 0xc42096e780, 0xc4202f46a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69228 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open.func2(0xc4209a7c30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:430 +0x64
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:431 +0x6d1

goroutine 68988 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4229087e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331440, 0xc421a4c280, 0xc420331420)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68609 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x7f4ce5286040, 0xc42048a0f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:102 +0x8e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420193ba0, 0xc42096e780, 0xc420198660)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68901 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc420e65e30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ae740, 0xc421a4c280, 0xc4202be400)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69060 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000060)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e1890)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203409e0, 0xc421a4c280, 0xc4201e8f60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69270 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x7f4ce5286040, 0xc420b3e480)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1064 +0x5e0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020f270, 0xc42056f5e0, 0xc42020f260)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69558 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4211ced80, 0xc420274eb0, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:143 +0x70a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc421583200, 0xc420274eb0, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:333 +0x7f
io.ReadAtLeast(0x30b4640, 0xc421583200, 0xc420274eb0, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30b4640, 0xc421583200, 0xc420274eb0, 0x5, 0x5, 0x0, 0x683bf2, 0x0)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc420274ea0, 0x7fffffff, 0xc420022c00, 0x2, 0x7, 0x6f1a4c, 0x6af2f8, 0x6afd24)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:243 +0xa0
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc420274ea0, 0x30c6f00, 0x3ef7888, 0xc421583200, 0x30bc000, 0x3ef7888, 0x216f080, 0xc42038d080, 0x7fffffff, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:339 +0x61
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc4218e2160, 0x216f080, 0xc42038d080, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:386 +0x20e
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc420279730, 0xc4213a46a8, 0x7f4ce5286040, 0xc42038d040)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:329 +0x86
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x30d06e0, 0xc420279730, 0xc4211ced00, 0xc42007a600, 0x7f4ce5286040, 0xc42038d020, 0x18, 0xc4213a4778)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:466 +0x5a
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1(0x7f4ce5286040, 0xc42038d020)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:483 +0x7e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420279780, 0xc421a4c280, 0xc42038cff0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68713 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x7f4ce5286040, 0xc42048a210)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1055 +0x65b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420260130, 0xc42096e780, 0xc420260120)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69562 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func3(0x30cea60, 0xc422886600, 0xc4215837a0, 0xc4218e2420)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:236 +0x52e
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:256 +0x1389

goroutine 69047 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000054)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420417da0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203408e0, 0xc421a4c280, 0xc4201e8d20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68760 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc420de6e10)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203372b0, 0xc42096e780, 0xc4203372a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69036 [select]:
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).Start.func1(0x7f4ce5286040, 0xc42038c990)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:115 +0x61c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42022f300, 0xc421a4c280, 0xc42022f2f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69026 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4221e0fc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203318c0, 0xc421a4c280, 0xc4203318b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68632 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc421583680, 0x23120e8, 0xc4221777a8, 0x30cea60)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:252 +0x220
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x7f4ce5287a60, 0xc421a47e60, 0x0, 0x0, 0x30c6f00, 0x3ef7888, 0x0, 0x0, 0x30bc000, 0x3ef7888, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:63 +0xc9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7f4ce5287a60, 0xc421a47e60, 0x22623f7, 0x1d, 0x217d880, 0xc422742180, 0x21684e0, 0xc420275ce0, 0xc4227a0300, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:265 +0xdf9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x7f4ce5287a60, 0xc421a47e60, 0x22623f7, 0x1d, 0x217d880, 0xc422742180, 0x21684e0, 0xc420275ce0, 0xc4227a0300, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:146 +0x217
github.com/cockroachdb/cockroach/pkg/rpc.(*heartbeatClient).Ping(0xc42283a018, 0x7f4ce5287a60, 0xc421a47e60, 0xc422742180, 0x0, 0x0, 0x0, 0x14bf546b246d59cc, 0x3232b6, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/heartbeat.pb.go:121 +0x107
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4205c9830, 0xc421d14000, 0xc4205746e0, 0x12, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:376 +0x508
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.2.1(0x7f4ce5286040, 0xc420e64270)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:297 +0x86
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ae010, 0xc42096e780, 0xc421484030)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68764 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c120)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337330, 0xc42096e780, 0xc420337320)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69510 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4213e0100, 0xc4201d1990, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:143 +0x70a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc420784b40, 0xc4201d1990, 0x5, 0x5, 0x0, 0x37, 0x37)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:333 +0x7f
io.ReadAtLeast(0x30b4640, 0xc420784b40, 0xc4201d1990, 0x5, 0x5, 0x5, 0x0, 0xc420397745, 0xc4203976c0)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30b4640, 0xc420784b40, 0xc4201d1990, 0x5, 0x5, 0x6808c1, 0xc420032800, 0x2)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4201d1980, 0x7fffffff, 0x2312068, 0xc4213e1880, 0xc4204fe398, 0x6ac7c2, 0x683e7d, 0xc41ff7f892)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:243 +0xa0
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4201d1980, 0x30c6f00, 0x3ef7888, 0xc420784b40, 0x30bc000, 0x3ef7888, 0x217d0a0, 0xc42100ed70, 0x7fffffff, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:339 +0x61
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc4204fe320, 0x217d0a0, 0xc42100ed70, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:613 +0x1d5
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipServer).Recv(0xc42022eed0, 0x300000000, 0x0, 0xc4228cfb10)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:230 +0x86
github.com/cockroachdb/cockroach/pkg/gossip.(Gossip_GossipServer).Recv-fm(0xc42095a340, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:163 +0x4b
github.com/cockroachdb/cockroach/pkg/gossip.(*server).gossipReceiver(0xc42095a300, 0x7f4ce5286040, 0xc42218eb70, 0xc42000e088, 0xc4213e0180, 0xc4228cff50, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:341 +0x6c3
github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip.func3.1(0x7f4ce5286040, 0xc42218eb70)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:163 +0x13a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42022ef10, 0xc421a4c280, 0xc4213e01c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69650 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc4217b9880, 0xc400000001, 0xc421bea360, 0xc4202a16c0, 0x30d06e0, 0xc420260a80, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:494 +0x9c4
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc4217b9880, 0x1, 0x7f4ce5286040, 0xc42033dc20, 0x1, 0x2, 0xc42056efa0, 0xc421bea360, 0xc4202a16c0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:439 +0x3d1
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc4217b9880, 0x7f4ce5286040, 0xc42033dc20, 0xc400000001, 0xc421bea360, 0xc4202a16c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:440 +0x103
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1(0x7f4ce5286040, 0xc42033dc20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:574 +0x7c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420289b20, 0xc42096e780, 0xc420b3fa40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68719 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000083)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420de6f60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261520, 0xc42096e780, 0xc4203acea0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69079 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000074)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38570)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380ba0, 0xc421a4c280, 0xc420214f40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69275 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420971320, 0xc4209cd300, 0xc4202524a0, 0x12, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:361 +0xb5b
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.2.1(0x7f4ce5286040, 0xc420b3e900)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:297 +0x86
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020f4b0, 0xc42056f5e0, 0xc420b3e8a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68825 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000007d)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e65380)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261a40, 0xc42096e780, 0xc4203ad7a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69012 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc422908b40)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331640, 0xc421a4c280, 0xc420331630)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69640 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4211cfa40, 0xc420198670, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:143 +0x70a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc4215837a0, 0xc420198670, 0x5, 0x5, 0xc4215c9aa8, 0x679899, 0xc422878ee0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:333 +0x7f
io.ReadAtLeast(0x30b4640, 0xc4215837a0, 0xc420198670, 0x5, 0x5, 0x5, 0xc422878ee0, 0x2076f00, 0x0)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30b4640, 0xc4215837a0, 0xc420198670, 0x5, 0x5, 0x117e087, 0x7d8857a3, 0xc422878ef0)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc420198660, 0x7fffffff, 0x1d0d7c0, 0x7f4cc52b5c08, 0xc4204f4740, 0x0, 0x0, 0xc4215c9bd0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:243 +0xa0
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc420198660, 0x30c6f00, 0x3ef7888, 0xc4215837a0, 0x30bc000, 0x3ef7888, 0x2166560, 0xc420bf78b0, 0x7fffffff, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:339 +0x61
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc4218e2420, 0x2166560, 0xc420bf78b0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:386 +0x20e
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc4202603c0, 0x7f4ce5286040, 0xc42218ed80, 0xc421b04780)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:194 +0x86
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0x30cf6c0, 0xc4202603c0, 0xc4209572b0, 0x7f4ce5286040, 0xc42218ed80, 0xc421b04780, 0xc422749768, 0x69c512)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:313 +0x43
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2(0x7f4ce5286040, 0xc42218ed80)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:321 +0xe6
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202607d0, 0xc42096e780, 0xc4211cfac0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68834 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x7f4ce5286040, 0xc420e654d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1351 +0x64e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261b10, 0xc42096e780, 0xc4203783c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68916 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func2(0x7f4ce5286040, 0xc420700d80)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:520 +0x95
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420295400, 0xc421a4c280, 0xc420353200)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68711 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4203ac140, 0x28, 0xc4211824b0, 0x3536b20, 0x0)
	<autogenerated>:9 +0x9f
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith(0xc4204a4018, 0x7f4ce5286040, 0xc420de62d0, 0xc42096e780, 0x30c6900, 0xc4203ac140, 0xc4211824b0, 0xc420de62d0, 0xc420dedea8)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:121 +0x61
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func9(0x7f4ce5286040, 0xc420de62d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:622 +0x28f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420260020, 0xc42096e780, 0xc421bec180)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69005 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000006b)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38390)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380a90, 0xc421a4c280, 0xc420214dc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69023 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4221e0de0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331860, 0xc421a4c280, 0xc420331850)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68797 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000061)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64d80)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261770, 0xc42096e780, 0xc4203ad300)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68945 [IO wait]:
net.runtime_pollWait(0x7f4ce653ca48, 0x72, 0x6fb68a)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc422803808, 0x72, 0x30adc98, 0xffffffffffffffff)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc422803808, 0xc421b7c000, 0x8000)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).Read(0xc4228037a0, 0xc421b7c000, 0x8000, 0x8000, 0x0, 0x30b8100, 0x30adc98)
	/usr/local/go/src/net/fd_unix.go:250 +0x22b
net.(*conn).Read(0xc420816190, 0xc421b7c000, 0x8000, 0x8000, 0x69fb14, 0xc4215e3a98, 0x6cbed0)
	/usr/local/go/src/net/net.go:181 +0x97
net.(*TCPConn).Read(0xc420816190, 0xc421b7c000, 0x8000, 0x8000, 0xc420020600, 0x67318c, 0xc420c259e0)
	<autogenerated>:72 +0x73
bufio.(*Reader).Read(0xc420c255c0, 0xc420ce0ab8, 0x9, 0x9, 0x6ef873, 0xc4208d9380, 0xffffffff)
	/usr/local/go/src/bufio/bufio.go:213 +0x7e7
io.ReadAtLeast(0x30af940, 0xc420c255c0, 0xc420ce0ab8, 0x9, 0x9, 0x9, 0x6d0f42, 0x6ac60c, 0xc420926a00)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30af940, 0xc420c255c0, 0xc420ce0ab8, 0x9, 0x9, 0xc4201b9aa0, 0xc4201b9aa0, 0xc4215e3e98)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420ce0ab8, 0x9, 0x9, 0x30af940, 0xc420c255c0, 0x0, 0x14e800000000, 0x14e8, 0x14e8)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420ce0a80, 0x30ba4c0, 0xc4201b9980, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x118
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc42033c480, 0xc4201b9980, 0xc4201b9900, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc420fdf500)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1043 +0x15b
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:251 +0x1294

goroutine 69488 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc4217b9880, 0x30d0740, 0xc4201a6620, 0x30b9d80, 0xc4217b9880)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:329 +0x32d
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x22125a0, 0xc4217b9880, 0x30cd8c0, 0xc420efc0a0, 0xc420e2b5d8, 0xc42137a5a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:378 +0xd6
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc4203355e0, 0x30ceac0, 0xc4200a4ee0, 0xc42137a5a0, 0xc4214a7e90, 0x30994a0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:842 +0xe22
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4203355e0, 0x30ceac0, 0xc4200a4ee0, 0xc42137a5a0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:936 +0x11c2
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420337500, 0xc4203355e0, 0x30ceac0, 0xc4200a4ee0, 0xc42137a5a0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:497 +0xb7
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:498 +0xb9

goroutine 69590 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4228e8840, 0xc420292f70, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:143 +0x70a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42137ac60, 0xc420292f70, 0x5, 0x5, 0x1d0d7c0, 0x7f4cbdb098c8, 0xc420fe01c8)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:333 +0x7f
io.ReadAtLeast(0x30b4640, 0xc42137ac60, 0xc420292f70, 0x5, 0x5, 0x5, 0xc420fe01c8, 0x6d104c, 0x6ef873)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30b4640, 0xc42137ac60, 0xc420292f70, 0x5, 0x5, 0xc420fe01c8, 0x4, 0x6d0f42)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc420292f60, 0x7fffffff, 0x1fa, 0x1fa, 0xc421d13d18, 0x69cbeb, 0x683e7d, 0xc41ffed6ad)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:243 +0xa0
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc420292f60, 0x30c6f00, 0x3ef7888, 0xc42137ac60, 0x30bc000, 0x3ef7888, 0x216efa0, 0xc420252a40, 0x7fffffff, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:339 +0x61
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc420efc460, 0x216efa0, 0xc420252a40, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:613 +0x1d5
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc4201a6fd0, 0x7f4ce5286040, 0xc4228e7380, 0xc4227f9200)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:397 +0x86
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x30d0740, 0xc4201a6fd0, 0xc42007a600, 0x7f4ce5286040, 0xc4228e7380, 0x18, 0xc420e30f78)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:300 +0x61
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1(0x7f4ce5286040, 0xc4228e7380)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:323 +0x75
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201a6ff0, 0xc421a4c280, 0xc4228e7350)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68718 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000081)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420de6ea0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261510, 0xc42096e780, 0xc4203ace80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69011 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4229086f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331620, 0xc421a4c280, 0xc420331610)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68774 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000089)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64bd0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202615a0, 0xc42096e780, 0xc4203acfa0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68958 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000004b)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420417bc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340850, 0xc421a4c280, 0xc4201e8bc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69086 [semacquire]:
sync.runtime_SemacquireMutex(0xc42050797c)
	/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc420507978)
	/usr/local/go/src/sync/mutex.go:87 +0xda
sync.(*RWMutex).Lock(0xc420507978)
	/usr/local/go/src/sync/rwmutex.go:86 +0x49
github.com/cockroachdb/cockroach/pkg/util/syncutil.(*RWMutex).Lock(0xc420507978)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:70 +0x3b
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleReplicatedEvalResult(0xc420507880, 0x7f4ce5286040, 0xc4201e3620, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_proposal.go:522 +0xe9
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleEvalResult(0xc420507880, 0x7f4ce5286040, 0xc4201e3620, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_proposal.go:794 +0x9b
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).processRaftCommand(0xc420507880, 0x7f4ce5286040, 0xc4201e3620, 0xc420294270, 0x8, 0x4b, 0x300000003, 0x3, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3706 +0x8d3
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc420507880, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2876 +0x10ca
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest(0xc4204d6800, 0x7f4ce5286040, 0xc421a38240, 0xc4219b2000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3110 +0x12ed
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue(0xc4204d6800, 0x1)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3389 +0x1f0
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:228 +0x30c
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38750)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380c20, 0xc421a4c280, 0xc4202150a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68759 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc420de6d80)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337290, 0xc42096e780, 0xc420337280)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69317 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc421c96a80)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1162 +0x6e6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:283 +0x1a4b

goroutine 68999 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000064)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a381e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203809f0, 0xc421a4c280, 0xc420214c80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68820 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000078)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e65260)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261970, 0xc42096e780, 0xc4203ad6e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68917 [IO wait]:
net.runtime_pollWait(0x7f4ce52cd438, 0x72, 0x30adc98)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc421d0bbf8, 0x72, 0x30adc98, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc421d0bbf8, 0xffffffffffffffff, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).accept(0xc421d0bb90, 0x0, 0x30b54c0, 0xc420387860)
	/usr/local/go/src/net/fd_unix.go:430 +0x2c0
net.(*TCPListener).accept(0xc42017e098, 0xc420700ef0, 0xc420700ed0, 0x69c512)
	/usr/local/go/src/net/tcpsock_posix.go:136 +0x51
net.(*TCPListener).Accept(0xc42017e098, 0xc420700e70, 0x2068c40, 0x308d090, 0x2157f40)
	/usr/local/go/src/net/tcpsock.go:228 +0x50
net/http.(*Server).Serve(0xc42027e8f0, 0x30c5e40, 0xc42017e098, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2643 +0x2a8
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func5(0x7f4ce5286040, 0xc420700db0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:543 +0x8f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420295410, 0xc421a4c280, 0xc420353220)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69081 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000007c)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a387e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380bc0, 0xc421a4c280, 0xc420214fe0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68807 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000006b)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64f90)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261860, 0xc42096e780, 0xc4203ad4a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68818 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000076)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e65200)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261950, 0xc42096e780, 0xc4203ad660)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69043 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000050)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420417d40)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203408a0, 0xc421a4c280, 0xc4201e8c80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68803 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000067)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64ea0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261810, 0xc42096e780, 0xc4203ad400)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69078 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000073)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38540)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380b90, 0xc421a4c280, 0xc420214f20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69561 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip(0xc4209572b0, 0x7f4ce5286040, 0xc42033d2c0, 0xc421b04780, 0x30cf6c0, 0xc4202603c0, 0xc42096e780, 0xc420260390, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:325 +0x4d7
github.com/cockroachdb/cockroach/pkg/gossip.(*client).startLocked.func1(0x7f4ce5286040, 0xc42033d2c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:132 +0x653
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420260350, 0xc42096e780, 0xc4211cf980)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69040 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420fe0120, 0xc420c224c0, 0xc420352fa0, 0x12, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:361 +0xb5b
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.2.1(0x7f4ce5286040, 0xc42048b230)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:297 +0x86
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42022f4d0, 0xc421a4c280, 0xc420b25e60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68893 [chan receive]:
database/sql.(*DB).connectionOpener(0xc421a4c140)
	/usr/local/go/src/database/sql/sql.go:837 +0x6e
created by database/sql.Open
	/usr/local/go/src/database/sql/sql.go:582 +0x318

goroutine 68986 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc422908750)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203313b0, 0xc421a4c280, 0xc4203313a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69007 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000006e)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38420)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380ab0, 0xc421a4c280, 0xc420214e00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68837 [IO wait]:
net.runtime_pollWait(0x7f4ce653ce08, 0x72, 0x6fb68a)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc42005c5a8, 0x72, 0x30adc98, 0xffffffffffffffff)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc42005c5a8, 0xc4216b0000, 0x8000)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).Read(0xc42005c540, 0xc4216b0000, 0x8000, 0x8000, 0x0, 0x30b8100, 0x30adc98)
	/usr/local/go/src/net/fd_unix.go:250 +0x22b
net.(*conn).Read(0xc420e78008, 0xc4216b0000, 0x8000, 0x8000, 0x21823c0, 0x0, 0x2312bc8)
	/usr/local/go/src/net/net.go:181 +0x97
net.(*TCPConn).Read(0xc420e78008, 0xc4216b0000, 0x8000, 0x8000, 0x1, 0x30b0040, 0xc420016110)
	<autogenerated>:72 +0x73
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4204fe960, 0xc4216b0000, 0x8000, 0x8000, 0x1d0ecc0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x183
bufio.(*Reader).Read(0xc420010e40, 0xc420335458, 0x9, 0x9, 0x7f4ce1e3db88, 0xc42218fda8, 0x0)
	/usr/local/go/src/bufio/bufio.go:213 +0x7e7
io.ReadAtLeast(0x30af940, 0xc420010e40, 0xc420335458, 0x9, 0x9, 0x9, 0xc421bc8a48, 0x6ef873, 0xc420f28680)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30af940, 0xc420010e40, 0xc420335458, 0x9, 0x9, 0xc4202ae9e0, 0x0, 0x1)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420335458, 0x9, 0x9, 0x30af940, 0xc420010e40, 0x0, 0xc400000000, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420335420, 0x0, 0x30b0040, 0xc420016110, 0xc4213e13c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x118
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc421070960, 0xc42218fe90, 0xc42218fe90, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420335ea0, 0xc421070b10, 0x2312020)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:307 +0x3c4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4203355e0, 0x30ceac0, 0xc420335ea0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:505 +0x1e8
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4203355e0, 0x30d0860, 0xc4204fe960, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:486 +0x5a6
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4203355e0, 0x30d0860, 0xc4204fe960)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:457 +0x5f4
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:426 +0x69c

goroutine 68809 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000006d)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64ff0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261880, 0xc42096e780, 0xc4203ad500)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68707 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func2(0x7f4ce5286040, 0xc42048a120)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:520 +0x95
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420193fd0, 0xc42096e780, 0xc4203ac3a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68881 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c450)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337610, 0xc42096e780, 0xc420337600)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68786 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000057)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421485170)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261670, 0xc42096e780, 0xc4203ad180)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69029 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4221e0d50)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331930, 0xc421a4c280, 0xc420331910)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68332 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc420de72f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420294370, 0xc42096e780, 0xc4202f4620)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68771 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000087)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420de7050)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261570, 0xc42096e780, 0xc4203acf40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68389 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc42045c900, 0x7f4ce5286000, 0xc4202a1dc0, 0x1dcd6500, 0x0, 0xc42045c948, 0x1, 0xc4202306e0, 0x3, 0x3, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1172 +0x2551
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc42045c900, 0x7f4ce5286000, 0xc4202a1dc0, 0x1, 0xc4202306e0, 0x3, 0x3, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:405 +0x3b4
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc42045c900, 0x7f4ce5286040, 0xc42168fce0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:474 +0x1a8
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc42045c900, 0x7f4ce5286040, 0xc42168fce0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:940 +0x4ec
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc42045c900, 0x7f4ce5286040, 0xc42168fce0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:810 +0xb78
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc42045c900, 0x7f4ce5286040, 0xc42168fce0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:624 +0x3ce
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4219cf790, 0x7f4ce5286040, 0xc42168fce0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:478 +0x490
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc420252980, 0x7f4ce5286040, 0xc42168fce0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:528 +0x228
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x7f4ce5286040, 0xc42168fce0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:464 +0x97
github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x7f4ce5286040, 0xc42168fce0, 0xc4215c1580, 0xc4203f0300, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:436 +0x13b
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc420252980, 0x7f4ce5286040, 0xc42168fce0, 0xc4203f0300, 0xc4215c16c0, 0x18)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:464 +0xf8
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Inc(0xc420252980, 0x7f4ce5286040, 0xc42168fce0, 0x2158d80, 0xc4205756e0, 0x1, 0x989680, 0x3b9aca00, 0x3ff8000000000000, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:289 +0xec
github.com/cockroachdb/cockroach/pkg/server.incVal(0x7f4ce5286040, 0xc42168fce0, 0xc420252980, 0xc4205228a0, 0xb, 0xb, 0x1, 0x30d3200, 0x3ef7888, 0x3ef7888)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:176 +0x206
github.com/cockroachdb/cockroach/pkg/server.allocateNodeID(0x7f4ce5286040, 0xc42168fce0, 0xc420252980, 0x224baa5, 0xd, 0x7f4ce5286040)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:148 +0x94
github.com/cockroachdb/cockroach/pkg/server.(*Node).initNodeID(0xc42106d200, 0xc400000000)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:340 +0x320
github.com/cockroachdb/cockroach/pkg/server.(*Node).initStores(0xc42106d200, 0x7f4ce5286040, 0xc4201b8690, 0xc420337890, 0x1, 0x1, 0xc42056f5e0, 0x0, 0x0, 0x7f4cd40c8308)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:517 +0x105d
github.com/cockroachdb/cockroach/pkg/server.(*Node).start(0xc42106d200, 0x7f4ce5286040, 0xc4201b8690, 0x30ba000, 0xc420253380, 0xc420337890, 0x1, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:377 +0x12f
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start(0xc42018b680, 0x7f4ce5286040, 0xc4201b8690, 0x7f4ce651c918, 0xc420016e28)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:692 +0x1942
github.com/cockroachdb/cockroach/pkg/server.(*TestServer).Start(0xc420193e50, 0x30b3780, 0xc420971290, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/testserver.go:294 +0x29e
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerRaw(0x30b3780, 0xc420971290, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:177 +0x144
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServer(0x30d4380, 0xc420e9e0d0, 0x30b3780, 0xc420971290, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:147 +0x74
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).doAddServer(0xc4200abe30, 0x30d4380, 0xc420e9e0d0, 0x30b3780, 0xc420971290, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:248 +0x177
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).AddServer(0xc4200abe30, 0x30d4380, 0xc420e9e0d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:221 +0xac
github.com/cockroachdb/cockroach/pkg/storage_test.TestGossipHandlesReplacedNode(0xc420e9e0d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/gossip_test.go:183 +0x7ae
testing.tRunner(0xc420e9e0d0, 0x2310bd0)
	/usr/local/go/src/testing/testing.go:657 +0x108
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:697 +0x544

goroutine 68336 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc420e65530)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202943b0, 0xc42096e780, 0xc4202f46c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68955 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000047)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4204179b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340810, 0xc421a4c280, 0xc4201e8b60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68877 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c1e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337590, 0xc42096e780, 0xc420337580)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69213 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func2(0x7f4ce5286040, 0xc4221c3410)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:113 +0x48f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020e850, 0xc42056f5e0, 0xc420252ce0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68789 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000005b)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64c60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202616a0, 0xc42096e780, 0xc4203ad200)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68882 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c480)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337630, 0xc42096e780, 0xc420337620)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69050 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000059)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e1770)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340910, 0xc421a4c280, 0xc4201e8d80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68900 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc420e65e00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ae730, 0xc421a4c280, 0xc4202be3a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69077 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000072)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38510)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380b80, 0xc421a4c280, 0xc420214f00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68838 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc420335ea0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:742 +0xa23
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:161 +0xdf2

goroutine 68909 [IO wait]:
net.runtime_pollWait(0x7f4ce653c208, 0x72, 0x6fb68a)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc42095c1b8, 0x72, 0x30adc98, 0xffffffffffffffff)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc42095c1b8, 0xc421604000, 0x8000)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).Read(0xc42095c150, 0xc421604000, 0x8000, 0x8000, 0x0, 0x30b8100, 0x30adc98)
	/usr/local/go/src/net/fd_unix.go:250 +0x22b
net.(*conn).Read(0xc420e78050, 0xc421604000, 0x8000, 0x8000, 0xe, 0x0, 0x2312bc8)
	/usr/local/go/src/net/net.go:181 +0x97
net.(*TCPConn).Read(0xc420e78050, 0xc421604000, 0x8000, 0x8000, 0x1, 0x30b0040, 0xc420016110)
	<autogenerated>:72 +0x73
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4204fe0a0, 0xc421604000, 0x8000, 0x8000, 0x1d0ecc0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x183
bufio.(*Reader).Read(0xc420a41f20, 0xc420ce0658, 0x9, 0x9, 0x7f4cd071e8c8, 0xc42038d598, 0x0)
	/usr/local/go/src/bufio/bufio.go:213 +0x7e7
io.ReadAtLeast(0x30af940, 0xc420a41f20, 0xc420ce0658, 0x9, 0x9, 0x9, 0xc4208f4a48, 0x6ef873, 0xc42055bba0)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30af940, 0xc420a41f20, 0xc420ce0658, 0x9, 0x9, 0xc4203a2750, 0x0, 0x1)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420ce0658, 0x9, 0x9, 0x30af940, 0xc420a41f20, 0x0, 0xc400000000, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420ce0620, 0x0, 0x30b0040, 0xc420016110, 0xc42038d6e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x118
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc420f60000, 0xc42038d6b0, 0xc42038d6b0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420ce07e0, 0xc420f60150, 0x2312020)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:307 +0x3c4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4200a41c0, 0x30ceac0, 0xc420ce07e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:505 +0x1e8
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4200a41c0, 0x30d0860, 0xc4204fe0a0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:486 +0x5a6
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4200a41c0, 0x30d0860, 0xc4204fe0a0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:457 +0x5f4
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:426 +0x69c

goroutine 69108 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc4227a0600)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1162 +0x6e6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:283 +0x1a4b

goroutine 69482 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc422886600)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1162 +0x6e6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:283 +0x1a4b

goroutine 68990 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc422908930)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203314d0, 0xc421a4c280, 0xc420331490)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68772 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000004a)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420de70b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261580, 0xc42096e780, 0xc4203acf60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69089 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000003f)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e65bc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380c50, 0xc421a4c280, 0xc420215120)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68874 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c2d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337520, 0xc42096e780, 0xc420337510)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68819 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000077)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e65230)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261960, 0xc42096e780, 0xc4203ad680)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69010 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc422908960)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331600, 0xc421a4c280, 0xc4203315d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68982 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1(0x7f4ce5286040, 0xc420d93bf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:661 +0x5cc
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203311b0, 0xc421a4c280, 0xc420346d00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68714 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x7f4ce5286040, 0xc420de6330)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1093 +0x415
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420260150, 0xc42096e780, 0xc420260140)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68887 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*Executor).Start.func1(0x7f4ce5286040, 0xc42038c540)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:366 +0x19d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203376e0, 0xc42096e780, 0xc4201d1cc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68866 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc420196780)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203373f0, 0xc42096e780, 0xc4203373e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69257 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc42081b200)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1162 +0x6e6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:283 +0x1a4b

goroutine 68717 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000080)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421484f30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261500, 0xc42096e780, 0xc4203ace40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69088 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000003e)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e65b90)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380c40, 0xc421a4c280, 0xc420215100)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69223 [IO wait]:
net.runtime_pollWait(0x7f4ce653d048, 0x72, 0x6fb68a)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc4216b81b8, 0x72, 0x30adc98, 0xffffffffffffffff)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc4216b81b8, 0xc42196e000, 0x8000)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).Read(0xc4216b8150, 0xc42196e000, 0x8000, 0x8000, 0x0, 0x30b8100, 0x30adc98)
	/usr/local/go/src/net/fd_unix.go:250 +0x22b
net.(*conn).Read(0xc42000e248, 0xc42196e000, 0x8000, 0x8000, 0x21823c0, 0x0, 0x2312bc8)
	/usr/local/go/src/net/net.go:181 +0x97
net.(*TCPConn).Read(0xc42000e248, 0xc42196e000, 0x8000, 0x8000, 0x1, 0x30b0040, 0xc420016110)
	<autogenerated>:72 +0x73
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4205286e0, 0xc42196e000, 0x8000, 0x8000, 0x1d0ecc0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x183
bufio.(*Reader).Read(0xc420ddfd40, 0xc4200a49d8, 0x9, 0x9, 0x7f4cd63a8e48, 0xc4210a0f68, 0x0)
	/usr/local/go/src/bufio/bufio.go:213 +0x7e7
io.ReadAtLeast(0x30af940, 0xc420ddfd40, 0xc4200a49d8, 0x9, 0x9, 0x9, 0xc421b07a48, 0x6ef873, 0xc420b23d40)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30af940, 0xc420ddfd40, 0xc4200a49d8, 0x9, 0x9, 0xc420330ec0, 0x0, 0x1)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4200a49d8, 0x9, 0x9, 0x30af940, 0xc420ddfd40, 0x0, 0xc400000000, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4200a49a0, 0x0, 0x30b0040, 0xc420016110, 0xc420397ac0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x118
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc420d93380, 0xc4210a1080, 0xc4210a1080, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc4200a4ee0, 0xc420d93830, 0x2312020)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:307 +0x3c4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4203355e0, 0x30ceac0, 0xc4200a4ee0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:505 +0x1e8
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4203355e0, 0x30d0860, 0xc4205286e0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:486 +0x5a6
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4203355e0, 0x30d0860, 0xc4205286e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:457 +0x5f4
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:426 +0x69c

goroutine 69132 [IO wait]:
net.runtime_pollWait(0x7f4ce52cd4f8, 0x72, 0x6fb68a)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc422802768, 0x72, 0x30adc98, 0xffffffffffffffff)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc422802768, 0xc4217f2000, 0x8000)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).Read(0xc422802700, 0xc4217f2000, 0x8000, 0x8000, 0x0, 0x30b8100, 0x30adc98)
	/usr/local/go/src/net/fd_unix.go:250 +0x22b
net.(*conn).Read(0xc4214a40a0, 0xc4217f2000, 0x8000, 0x8000, 0x69cbeb, 0xc42001e000, 0x0)
	/usr/local/go/src/net/net.go:181 +0x97
net.(*TCPConn).Read(0xc4214a40a0, 0xc4217f2000, 0x8000, 0x8000, 0x0, 0xc4204762f0, 0x10)
	<autogenerated>:72 +0x73
bufio.(*Reader).Read(0xc420011a40, 0xc42039eff8, 0x9, 0x9, 0x0, 0x0, 0xc421c22be0)
	/usr/local/go/src/bufio/bufio.go:213 +0x7e7
io.ReadAtLeast(0x30af940, 0xc420011a40, 0xc42039eff8, 0x9, 0x9, 0x9, 0xc4206331e0, 0xffffffff, 0xc42026a190)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30af940, 0xc420011a40, 0xc42039eff8, 0x9, 0x9, 0xd70eb8, 0xc4211d6430, 0x0)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc42039eff8, 0x9, 0x9, 0x30af940, 0xc420011a40, 0x0, 0xc400000000, 0xc420934ea0, 0x30b0040)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42039efc0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x118
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc4221c2bd0, 0xc420216db0, 0xc420216d00, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc4211d6300)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1043 +0x15b
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:251 +0x1294

goroutine 69096 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x7f4ce5286040, 0xc421a38150)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1351 +0x64e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380cf0, 0xc421a4c280, 0xc4202bd720)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68785 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000055)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64c30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261660, 0xc42096e780, 0xc4203ad160)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69094 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x7f4ce5286040, 0xc420e65cb0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1351 +0x64e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380cd0, 0xc421a4c280, 0xc4202bd680)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68822 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000007a)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e652c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202619a0, 0xc42096e780, 0xc4203ad720)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68765 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4201968a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337350, 0xc42096e780, 0xc420337340)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69210 [chan receive]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1(0x7f4ce5286040, 0xc42050fda0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:176 +0x95
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020e010, 0xc42056f5e0, 0xc420252500)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68708 [IO wait]:
net.runtime_pollWait(0x7f4ce653c988, 0x72, 0x30adc98)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc421d0a378, 0x72, 0x30adc98, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc421d0a378, 0xffffffffffffffff, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).accept(0xc421d0a310, 0x0, 0x30b54c0, 0xc4202be800)
	/usr/local/go/src/net/fd_unix.go:430 +0x2c0
net.(*TCPListener).accept(0xc4204a4038, 0xc420de6230, 0xc420de6210, 0x69c512)
	/usr/local/go/src/net/tcpsock_posix.go:136 +0x51
net.(*TCPListener).Accept(0xc4204a4038, 0xc420de61e0, 0x2068c40, 0x308d090, 0x2157f40)
	/usr/local/go/src/net/tcpsock.go:228 +0x50
net/http.(*Server).Serve(0xc420702160, 0x30c5e40, 0xc4204a4038, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2643 +0x2a8
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func5(0x7f4ce5286040, 0xc420b3ffb0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:543 +0x8f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420193ff0, 0xc42096e780, 0xc4203ac3c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68781 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000054)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64c00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261620, 0xc42096e780, 0xc4203ad0a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69553 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc420d25440, 0x23120e8, 0xc4215db598, 0x30cea60)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:252 +0x220
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x7f4ce5286040, 0xc4223d37a0, 0x0, 0x0, 0x30c6f00, 0x3ef7888, 0x0, 0x0, 0x30bc000, 0x3ef7888, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:63 +0xc9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7f4ce5286040, 0xc4223d37a0, 0x2269a28, 0x21, 0x221faa0, 0xc42001ce68, 0x21926e0, 0xc421844ba0, 0xc421c96900, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:265 +0xdf9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x7f4ce5286040, 0xc4223d37a0, 0x2269a28, 0x21, 0x221faa0, 0xc42001ce68, 0x21926e0, 0xc421844ba0, 0xc421c96900, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:146 +0x217
github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc420e78060, 0x7f4ce5286040, 0xc4223d37a0, 0xc42001ce68, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1854 +0x107
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc420230780, 0xc42001ce40, 0x7f4ce5286040, 0xc4223d37a0, 0x149f55b, 0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:261 +0x253
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420230780, 0xc42001ce40, 0x7f4ce5286040, 0xc4223d37a0, 0x30d3200, 0x3ef7888, 0xc42093d0e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:270 +0xdd
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:283 +0x236

goroutine 69314 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420971320, 0xc421bced00, 0xc4201db140, 0x12, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:361 +0xb5b
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.2.1(0x7f4ce5286040, 0xc42048a630)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:297 +0x86
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203ae6e0, 0xc42056f5e0, 0xc42048a360)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69004 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000069)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38330)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380a60, 0xc421a4c280, 0xc420214da0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68903 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc420e65e60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ae760, 0xc421a4c280, 0xc4202be460)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69274 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc421324b60)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:859 +0x7cf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc421324b60)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:604 +0x267
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:605 +0xa4d

goroutine 68813 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000071)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e650e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202618e0, 0xc42096e780, 0xc4203ad5a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68657 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*LeaseManager).RefreshLeases.func1(0x7f4ce5286040, 0xc4207442d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/lease.go:1240 +0xf43
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202493c0, 0xc42096e780, 0xc42094c210)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68831 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func2(0x7f4ce5286040, 0xc420de7260)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1222 +0x19d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261ae0, 0xc42096e780, 0xc4203ad840)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69056 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000005b)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e17d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340970, 0xc421a4c280, 0xc4201e8ea0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68883 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c4e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337650, 0xc42096e780, 0xc420337640)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69107 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc4227a0600)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1116 +0x912
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:281 +0x1954

goroutine 68833 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x7f4ce5286040, 0xc421485260)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1351 +0x64e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261b00, 0xc42096e780, 0xc420378320)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68821 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000079)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e65290)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261990, 0xc42096e780, 0xc4203ad700)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69194 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc420b742a0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:742 +0xa23
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:161 +0xdf2

goroutine 68853 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc420fde180)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1162 +0x6e6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:283 +0x1a4b

goroutine 69307 [select]:
github.com/cockroachdb/cockroach/pkg/kv.grpcTransportFactory.func1(0x7f4ce5286040, 0xc4223d3f20)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport_race.go:96 +0x3da
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202890e0, 0xc42056f5e0, 0xc4202890c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68892 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc42090b1e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:859 +0x7cf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc42090b1e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:604 +0x267
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:605 +0xa4d

goroutine 68720 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000084)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420de6f90)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261530, 0xc42096e780, 0xc4203acec0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69009 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000006f)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38480)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380b40, 0xc421a4c280, 0xc420214e60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69278 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc42167a0e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:742 +0xa23
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:161 +0xdf2

goroutine 69133 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc4211d6300)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1116 +0x912
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:281 +0x1954

goroutine 69316 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc421c96a80)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1116 +0x912
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:281 +0x1954

goroutine 69030 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4221e0f90)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331960, 0xc421a4c280, 0xc420331940)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69507 [semacquire]:
sync.runtime_Semacquire(0xc420507984)
	/usr/local/go/src/runtime/sema.go:47 +0x34
sync.(*RWMutex).RLock(0xc420507978)
	/usr/local/go/src/sync/rwmutex.go:43 +0x95
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Desc(0xc420507880, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1154 +0x54
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func2(0xc42196c0c8, 0xc420fa61e0, 0xc420507880, 0x7f4ce5286040, 0xc42218e690, 0xc4202d0310, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1120 +0x5a7
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc420507880, 0x7f4ce5286040, 0xc42218e690, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1125 +0x25d
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeReadOnlyBatch(0xc420507880, 0x7f4ce5286040, 0xc42218e690, 0x14bf546bcbfa56f6, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2014 +0xaa8
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420507880, 0x7f4ce5286040, 0xc42218e690, 0x14bf546bcbfa56f6, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1428 +0x576
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4204d6800, 0x7f4ce5286040, 0xc42218e660, 0x14bf546bcbfa56f6, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2555 +0x9e0
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42007a700, 0x7f4ce5286040, 0xc42218e630, 0x0, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x24b
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x7f4ce5286040, 0xc42218e630, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:843 +0x20f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421a4c280, 0x7f4ce5286040, 0xc42218e600, 0xc42196d7d8, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:272 +0x14f
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4200b4000, 0x7f4ce5286040, 0xc42218e600, 0xc4218b89e8, 0xc42218e600, 0x1140100, 0x226743b)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:854 +0x1c1
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4200b4000, 0x7f4ce5286040, 0xc42218e600, 0xc4218b89e8, 0x1f, 0x680d68, 0xc420d8e500)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:871 +0xb8
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc422838dc0, 0xc4218b89c0, 0x7f4ce5286040, 0xc42218e5a0, 0x149f55b, 0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:257 +0x735
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422838dc0, 0xc4218b89c0, 0x7f4ce5286040, 0xc42218e5a0, 0x30d3200, 0x3ef7888, 0xc421bf4960)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:270 +0xdd
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:283 +0x236

goroutine 69042 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000004e)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4203ba600)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340890, 0xc421a4c280, 0xc4201e8c60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68961 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000004d)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420417cb0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340880, 0xc421a4c280, 0xc4201e8c40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69001 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000066)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38270)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380a30, 0xc421a4c280, 0xc420214ce0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68910 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc420ce07e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:742 +0xa23
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:161 +0xdf2

goroutine 69022 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4221e0e10)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331840, 0xc421a4c280, 0xc420331820)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68337 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc420e65560)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202943c0, 0xc42096e780, 0xc4202f46e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68808 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000006c)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64fc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261870, 0xc42096e780, 0xc4203ad4c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68851 [IO wait]:
net.runtime_pollWait(0x7f4ce52cd5b8, 0x72, 0x6fb68a)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc421d0a688, 0x72, 0x30adc98, 0xffffffffffffffff)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc421d0a688, 0xc4222cc000, 0x8000)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).Read(0xc421d0a620, 0xc4222cc000, 0x8000, 0x8000, 0x0, 0x30b8100, 0x30adc98)
	/usr/local/go/src/net/fd_unix.go:250 +0x22b
net.(*conn).Read(0xc42283a020, 0xc4222cc000, 0x8000, 0x8000, 0x69cbeb, 0xc420021900, 0x0)
	/usr/local/go/src/net/net.go:181 +0x97
net.(*TCPConn).Read(0xc42283a020, 0xc4222cc000, 0x8000, 0x8000, 0x0, 0xc420330f60, 0x10)
	<autogenerated>:72 +0x73
bufio.(*Reader).Read(0xc4207e6ae0, 0xc42039e738, 0x9, 0x9, 0x0, 0x0, 0xc4221f8be0)
	/usr/local/go/src/bufio/bufio.go:213 +0x7e7
io.ReadAtLeast(0x30af940, 0xc4207e6ae0, 0xc42039e738, 0x9, 0x9, 0x9, 0xc420ed7ba0, 0xffffffff, 0xc42241bb80)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30af940, 0xc4207e6ae0, 0xc42039e738, 0x9, 0x9, 0xd70eb8, 0xc420fde2b0, 0x0)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc42039e738, 0x9, 0x9, 0x30af940, 0xc4207e6ae0, 0x0, 0xc400000000, 0xc420934d80, 0x30b0040)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42039e700, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x118
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc420e65650, 0xc4210a1440, 0xc4210a1400, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc420fde180)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1043 +0x15b
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:251 +0x1294

goroutine 68879 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c420)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203375d0, 0xc42096e780, 0xc4203375c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69272 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip(0xc420956680, 0x7f4ce5286040, 0xc420b3e810, 0xc4219b2900, 0x30cf6c0, 0xc4202fad90, 0xc42056f5e0, 0xc42020f440, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:325 +0x4d7
github.com/cockroachdb/cockroach/pkg/gossip.(*client).startLocked.func1(0x7f4ce5286040, 0xc420b3e810)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:132 +0x653
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020f400, 0xc42056f5e0, 0xc4209cd240)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68889 [IO wait]:
net.runtime_pollWait(0x7f4ce653c388, 0x72, 0x30adc98)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc421d0a308, 0x72, 0x30adc98, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc421d0a308, 0xffffffffffffffff, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).accept(0xc421d0a2a0, 0x0, 0x30b54c0, 0xc420253a60)
	/usr/local/go/src/net/fd_unix.go:430 +0x2c0
net.(*TCPListener).accept(0xc4204a4020, 0xc420fd1e38, 0x6a695c, 0xc420fd1e10)
	/usr/local/go/src/net/tcpsock_posix.go:136 +0x51
net.(*TCPListener).Accept(0xc4204a4020, 0x23112f8, 0xc421bec080, 0x30d0980, 0xc4204a43c8)
	/usr/local/go/src/net/tcpsock.go:228 +0x50
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc421bec080, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0xdd
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func10.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:741 +0x47
sync.(*Once).Do(0xc420193ed0, 0xc4208c4750)
	/usr/local/go/src/sync/once.go:44 +0xe2
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func10(0x7f4ce5286040, 0xc4206aa9f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:742 +0xc3
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331720, 0xc42096e780, 0xc42026ca40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69021 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4203ba330)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331810, 0xc421a4c280, 0xc420331800)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68959 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000004a)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4203ba570)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340860, 0xc421a4c280, 0xc4201e8c00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68921 [chan receive]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x7f4ce5286040, 0xc4207014d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:386 +0x94
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420295540, 0xc421a4c280, 0xc420353320)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68960 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000004c)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420417c80)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340870, 0xc421a4c280, 0xc4201e8c20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69212 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*LeaseManager).RefreshLeases.func1(0x7f4ce5286040, 0xc4221c3440)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/lease.go:1240 +0xf43
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020e830, 0xc42056f5e0, 0xc420363920)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69087 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000007b)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a387b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380c30, 0xc421a4c280, 0xc4202150c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69484 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).controller(0xc420ce1180)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:742 +0xa23
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:161 +0xdf2

goroutine 69214 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x7f4ce5286040, 0xc421a38e10)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:102 +0x8e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020eff0, 0xc42056f5e0, 0xc420253200)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69085 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000007a)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38720)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380c10, 0xc421a4c280, 0xc420215080)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68957 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000049)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420417b60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340840, 0xc421a4c280, 0xc4201e8ba0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68919 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4203530c0, 0x2311fe8, 0xc4200a41c0, 0x30d0860, 0xc420d23ae0)
	<autogenerated>:9 +0x9f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve(0xc4200a41c0, 0x30c6900, 0xc4203530c0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:396 +0x1f8
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func7(0x7f4ce5286040, 0xc420700cc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:558 +0x8d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420295440, 0xc421a4c280, 0xc420353240)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68956 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000048)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420417a70)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340820, 0xc421a4c280, 0xc4201e8b80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69076 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000075)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a385d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380b70, 0xc421a4c280, 0xc420214ee0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69534 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc420bbaa80)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1162 +0x6e6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:283 +0x1a4b

goroutine 69083 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000078)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38690)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380be0, 0xc421a4c280, 0xc420215020)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68827 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000007f)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e653e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261a60, 0xc42096e780, 0xc4203ad7e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69128 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc42105cea0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:859 +0x7cf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc42105cea0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:604 +0x267
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:605 +0xa4d

goroutine 68802 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000066)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64e70)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261800, 0xc42096e780, 0xc4203ad3e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69574 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func2(0xc420d300b8, 0xc4227b0960, 0xc420507880, 0x7f4ce5286040, 0xc421c395f0, 0xc4227e0a10, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1075 +0x72c
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc420507880, 0x7f4ce5286040, 0xc421c395f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1125 +0x25d
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc420507880, 0x7f4ce5286040, 0xc421c395f0, 0x14bf546b70591396, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0xc4214a04e0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2204 +0x1298
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc420507880, 0x7f4ce5286040, 0xc421c395f0, 0x14bf546b70591396, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0xc4214a04e0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2091 +0xa9
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420507880, 0x7f4ce5286040, 0xc421c395f0, 0x14bf546b70591396, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0xc4214a04e0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1425 +0x28e
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4204d6800, 0x7f4ce5286040, 0xc421c395c0, 0x14bf546b70591396, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0xc4214a04e0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2555 +0x9e0
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42007a700, 0x7f4ce5286040, 0xc421c39590, 0x0, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0xc4214a04e0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x24b
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x7f4ce5286040, 0xc421c39590, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:843 +0x20f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421a4c280, 0x7f4ce5286040, 0xc421c39560, 0xc420d317d8, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:272 +0x14f
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4200b4000, 0x7f4ce5286040, 0xc421c39560, 0xc4216ec328, 0xc421c39560, 0x1140100, 0x226743b)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:854 +0x1c1
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4200b4000, 0x7f4ce5286040, 0xc421c39560, 0xc4216ec328, 0x1f, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:871 +0xb8
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1.1(0xc4203790e0, 0xc4216ec300, 0x7f4ce5286040, 0xc421c39500, 0xc421d1ef80, 0xd92311, 0xc421d1ef90)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:257 +0x735
github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4203790e0, 0xc4216ec300, 0x7f4ce5286040, 0xc421c39500, 0x30d3200, 0x3ef7888, 0xc4227b0480)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:270 +0xdd
created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:283 +0x236

goroutine 68782 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000050)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421485050)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261630, 0xc42096e780, 0xc4203ad100)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69054 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000005e)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420417e30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340950, 0xc421a4c280, 0xc4201e8e20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69018 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*SchemaChangeManager).Start.func1(0x7f4ce5286040, 0xc422908c60)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/schema_changer.go:807 +0x1f0c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331740, 0xc421a4c280, 0xc420346e20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69015 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc422908b10)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203316b0, 0xc421a4c280, 0xc4203316a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69528 [IO wait]:
net.runtime_pollWait(0x7f4ce653c508, 0x72, 0x6fb68a)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc421b84df8, 0x72, 0x30adc98, 0xffffffffffffffff)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc421b84df8, 0xc420834000, 0x8000)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).Read(0xc421b84d90, 0xc420834000, 0x8000, 0x8000, 0x0, 0x30b8100, 0x30adc98)
	/usr/local/go/src/net/fd_unix.go:250 +0x22b
net.(*conn).Read(0xc420e78088, 0xc420834000, 0x8000, 0x8000, 0x94acdd, 0xc421d0af50, 0x16)
	/usr/local/go/src/net/net.go:181 +0x97
net.(*TCPConn).Read(0xc420e78088, 0xc420834000, 0x8000, 0x8000, 0xc4212a6b28, 0x6ac6f2, 0x1d0b200)
	<autogenerated>:72 +0x73
bufio.(*Reader).Read(0xc421cf4600, 0xc4202c62d8, 0x9, 0x9, 0x6a01c5, 0xc420239800, 0xc420020600)
	/usr/local/go/src/bufio/bufio.go:213 +0x7e7
io.ReadAtLeast(0x30af940, 0xc421cf4600, 0xc4202c62d8, 0x9, 0x9, 0x9, 0xc42105d380, 0xc421cf4300, 0xc420e2fe90)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30af940, 0xc421cf4600, 0xc4202c62d8, 0x9, 0x9, 0x2, 0x2, 0x1)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4202c62d8, 0x9, 0x9, 0x30af940, 0xc421cf4600, 0x0, 0x0, 0x1d0b200, 0x7f4cd7d1b088)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4202c62a0, 0x6cc760, 0x6ca130, 0xc421cf4420, 0x2)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x118
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc420b251d0, 0x98c384, 0x98c301, 0xc420e2ffa8, 0x98c30e)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc420bba900)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1028 +0x7e
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:251 +0x1294

goroutine 69586 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc4217b9880, 0xc400000003, 0xc422870720, 0xc4228e8500, 0x30d06e0, 0xc4201a6f10, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:494 +0x9c4
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc4217b9880, 0x3, 0x7f4ce5286040, 0xc4228e6fc0, 0x0, 0x0, 0xc420efc140, 0xc422870720, 0xc4228e8500, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:439 +0x3d1
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc4217b9880, 0x7f4ce5286040, 0xc4228e6fc0, 0xc400000003, 0xc422870720, 0xc4228e8500)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:440 +0x103
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1(0x7f4ce5286040, 0xc4228e6fc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:574 +0x7c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201a6e50, 0xc42096e780, 0xc4228e6e10)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68954 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000046)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420417920)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340800, 0xc421a4c280, 0xc4201e8b40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69266 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x7f4ce5286040, 0xc421a38fc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:547 +0xd2
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020f130, 0xc42056f5e0, 0xc4209cce00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68796 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000060)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420de7170)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261760, 0xc42096e780, 0xc4203ad2e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69003 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000068)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a382d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380a50, 0xc421a4c280, 0xc420214d40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68876 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc421070420)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337570, 0xc42096e780, 0xc420337560)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69008 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000006d)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a383f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380ac0, 0xc421a4c280, 0xc420214e40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68710 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4203ac180, 0x2311fe8, 0xc4203355e0, 0x30d0860, 0xc42056fea0)
	<autogenerated>:9 +0x9f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve(0xc4203355e0, 0x30c6900, 0xc4203ac180, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:396 +0x1f8
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func7(0x7f4ce5286040, 0xc420de6240)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:558 +0x8d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420260010, 0xc42096e780, 0xc4203ac3e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68981 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1(0x7f4ce5286040, 0xc42050f260)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:702 +0x2bd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203311a0, 0xc421a4c280, 0xc420346ce0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68975 [chan receive]:
database/sql.(*DB).connectionOpener(0xc421a4d7c0)
	/usr/local/go/src/database/sql/sql.go:837 +0x6e
created by database/sql.Open
	/usr/local/go/src/database/sql/sql.go:582 +0x318

goroutine 69243 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4209cd640, 0xc4205750f0, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:143 +0x70a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42077e480, 0xc4205750f0, 0x5, 0x5, 0x0, 0x37, 0x37)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:333 +0x7f
io.ReadAtLeast(0x30b4640, 0xc42077e480, 0xc4205750f0, 0x5, 0x5, 0x5, 0x0, 0xc421bce345, 0xc421bce2c0)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30b4640, 0xc42077e480, 0xc4205750f0, 0x5, 0x5, 0x6808c1, 0xc420032c00, 0x2)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4205750e0, 0x7fffffff, 0x2312068, 0xc4203dfa00, 0xc4217b6898, 0x6ac7c2, 0x683e7d, 0xc41ff2244c)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:243 +0xa0
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4205750e0, 0x30c6f00, 0x3ef7888, 0xc42077e480, 0x30bc000, 0x3ef7888, 0x217d0a0, 0xc421bb7630, 0x7fffffff, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:339 +0x61
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc4217b6820, 0x217d0a0, 0xc421bb7630, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:613 +0x1d5
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipServer).Recv(0xc4202886b0, 0xc400000000, 0x1197ef7, 0xc421697b10)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:230 +0x86
github.com/cockroachdb/cockroach/pkg/gossip.(Gossip_GossipServer).Recv-fm(0xc4204f4740, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:163 +0x4b
github.com/cockroachdb/cockroach/pkg/gossip.(*server).gossipReceiver(0xc4204f4700, 0x7f4ce5286040, 0xc42168f530, 0xc42283a1e0, 0xc4203dffc0, 0xc421697f50, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:341 +0x6c3
github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip.func3.1(0x7f4ce5286040, 0xc42168f530)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:163 +0x13a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420288740, 0xc42096e780, 0xc421bce200)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68757 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteSummaries.func1(0x7f4ce5286040, 0xc420de6d50)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:739 +0x3b5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337250, 0xc42096e780, 0xc4201d1c20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68904 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc420e65ec0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ae770, 0xc421a4c280, 0xc4202be480)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69057 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000005a)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e17a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340990, 0xc421a4c280, 0xc4201e8ec0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69255 [IO wait]:
net.runtime_pollWait(0x7f4ce653c688, 0x72, 0x6fb68a)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc4200a7db8, 0x72, 0x30adc98, 0xffffffffffffffff)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc4200a7db8, 0xc421c98000, 0x8000)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).Read(0xc4200a7d50, 0xc421c98000, 0x8000, 0x8000, 0x0, 0x30b8100, 0x30adc98)
	/usr/local/go/src/net/fd_unix.go:250 +0x22b
net.(*conn).Read(0xc420e78098, 0xc421c98000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:181 +0x97
net.(*TCPConn).Read(0xc420e78098, 0xc421c98000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	<autogenerated>:72 +0x73
bufio.(*Reader).Read(0xc420805f20, 0xc4201ec8f8, 0x9, 0x9, 0x6a01c5, 0xc420d8c000, 0xc420026600)
	/usr/local/go/src/bufio/bufio.go:213 +0x7e7
io.ReadAtLeast(0x30af940, 0xc420805f20, 0xc4201ec8f8, 0x9, 0x9, 0x9, 0xc42110d860, 0xc420d069c0, 0xc42112b690)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30af940, 0xc420805f20, 0xc4201ec8f8, 0x9, 0x9, 0x2, 0x2, 0x1)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4201ec8f8, 0x9, 0x9, 0x30af940, 0xc420805f20, 0x0, 0x0, 0x1d0b200, 0x7f4cd11c1d48)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4201ec8c0, 0x6cc760, 0x6ca130, 0xc420d06a20, 0x2)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x118
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc420a75830, 0x98c384, 0x98c301, 0xc42112b7a8, 0x98c30e)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc42081b200)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1028 +0x7e
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:251 +0x1294

goroutine 69556 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc42007a600, 0xc400000002, 0xc421a46900, 0xc4211ced00, 0x30d06e0, 0xc420279730, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:494 +0x9c4
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc42007a600, 0x2, 0x7f4ce5286040, 0xc42038cf00, 0x0, 0x0, 0xc42056e820, 0xc421a46900, 0xc4211ced00, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:439 +0x3d1
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc42007a600, 0x7f4ce5286040, 0xc42038cf00, 0xc400000002, 0xc421a46900, 0xc4211ced00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:440 +0x103
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1(0x7f4ce5286040, 0xc42038cf00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:574 +0x7c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202796e0, 0xc421a4c280, 0xc42038ced0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68998 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000067)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a382a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203809e0, 0xc421a4c280, 0xc420214c40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69277 [IO wait]:
net.runtime_pollWait(0x7f4ce653c808, 0x72, 0x6fb68a)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc4216eab58, 0x72, 0x30adc98, 0xffffffffffffffff)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc4216eab58, 0xc420fc2000, 0x8000)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).Read(0xc4216eaaf0, 0xc420fc2000, 0x8000, 0x8000, 0x0, 0x30b8100, 0x30adc98)
	/usr/local/go/src/net/fd_unix.go:250 +0x22b
net.(*conn).Read(0xc4204a43c8, 0xc420fc2000, 0x8000, 0x8000, 0x21823c0, 0x0, 0x2312bc8)
	/usr/local/go/src/net/net.go:181 +0x97
net.(*TCPConn).Read(0xc4204a43c8, 0xc420fc2000, 0x8000, 0x8000, 0x1, 0x30b0040, 0xc420016110)
	<autogenerated>:72 +0x73
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42056fea0, 0xc420fc2000, 0x8000, 0x8000, 0x1d0ecc0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x183
bufio.(*Reader).Read(0xc420c59bc0, 0xc4202c7df8, 0x9, 0x9, 0x7f4ce3bac188, 0xc420b24a28, 0x0)
	/usr/local/go/src/bufio/bufio.go:213 +0x7e7
io.ReadAtLeast(0x30af940, 0xc420c59bc0, 0xc4202c7df8, 0x9, 0x9, 0x9, 0xc4215dea48, 0x6ef873, 0xc421118b60)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30af940, 0xc420c59bc0, 0xc4202c7df8, 0x9, 0x9, 0xc420248c70, 0x0, 0x1)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4202c7df8, 0x9, 0x9, 0x30af940, 0xc420c59bc0, 0x0, 0xc400000000, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4202c7dc0, 0x0, 0x30b0040, 0xc420016110, 0xc420c22940)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x118
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc420b3eba0, 0xc420b24b40, 0xc420b24b40, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc42167a0e0, 0xc420b3ecf0, 0x2312020)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:307 +0x3c4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4203355e0, 0x30ceac0, 0xc42167a0e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:505 +0x1e8
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4203355e0, 0x30d0860, 0xc42056fea0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:486 +0x5a6
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4203355e0, 0x30d0860, 0xc42056fea0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:457 +0x5f4
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:426 +0x69c

goroutine 68655 [chan receive]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1(0x7f4ce5286040, 0xc4207442a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:176 +0x95
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420248c70, 0xc42096e780, 0xc420574740)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68724 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc42085e1a0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:859 +0x7cf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc42085e1a0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:604 +0x267
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:605 +0xa4d

goroutine 69310 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4227419e0, 0x23120e8, 0xc4216917a8, 0x30cea60)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:252 +0x220
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x7f4ce5287a60, 0xc420fa77a0, 0x0, 0x0, 0x30c6f00, 0x3ef7888, 0x0, 0x0, 0x30bc000, 0x3ef7888, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:63 +0xc9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7f4ce5287a60, 0xc420fa77a0, 0x22623f7, 0x1d, 0x217d880, 0xc421bced80, 0x21684e0, 0xc4203611e0, 0xc421b05680, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:265 +0xdf9
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x7f4ce5287a60, 0xc420fa77a0, 0x22623f7, 0x1d, 0x217d880, 0xc421bced80, 0x21684e0, 0xc4203611e0, 0xc421b05680, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:146 +0x217
github.com/cockroachdb/cockroach/pkg/rpc.(*heartbeatClient).Ping(0xc42283a328, 0x7f4ce5287a60, 0xc420fa77a0, 0xc421bced80, 0x0, 0x0, 0x0, 0x21bcea80, 0x8, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/heartbeat.pb.go:121 +0x107
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420971320, 0xc421bcea40, 0xc4205757e0, 0x12, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:376 +0x508
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.2.1(0x7f4ce5286040, 0xc42048a240)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:297 +0x86
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203ae610, 0xc42056f5e0, 0xc42048a030)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69092 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startCoalescedHeartbeatsLoop.func1(0x7f4ce5286040, 0xc420e65ce0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3517 +0x1c5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380cb0, 0xc421a4c280, 0xc420380ca0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68690 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func2(0x7f4ce5286040, 0xc42094c270)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:113 +0x48f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202493d0, 0xc42096e780, 0xc420574da0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69000 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000065)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38210)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380a20, 0xc421a4c280, 0xc420214ca0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69283 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc420fdf500)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1162 +0x6e6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:283 +0x1a4b

goroutine 68783 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000052)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4214850b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261640, 0xc42096e780, 0xc4203ad120)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68754 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1(0x7f4ce5286040, 0xc420de6750)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:661 +0x5cc
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203370f0, 0xc42096e780, 0xc4201d1b00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69017 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc422908c30)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331720, 0xc421a4c280, 0xc420331700)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68756 [select]:
github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1(0x7f4ce5286040, 0xc4214a67b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/ts/db.go:90 +0x1c7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337240, 0xc42096e780, 0xc420337230)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69509 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip(0xc42095a300, 0x30cf720, 0xc42022eed0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:200 +0x772
github.com/cockroachdb/cockroach/pkg/gossip._Gossip_Gossip_Handler(0x220b100, 0xc42095a300, 0x30cd8c0, 0xc4204fe320, 0xc421a33dd8, 0xc420784b40)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:211 +0xd6
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc4200a41c0, 0x30ceac0, 0xc420ce1180, 0xc420784b40, 0xc421484ab0, 0x3091180, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:842 +0xe22
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4200a41c0, 0x30ceac0, 0xc420ce1180, 0xc420784b40, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:936 +0x11c2
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420351940, 0xc4200a41c0, 0x30ceac0, 0xc420ce1180, 0xc420784b40)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:497 +0xb7
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:498 +0xb9

goroutine 68987 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc422908780)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331400, 0xc421a4c280, 0xc4203313f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68839 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4205c9830, 0xc42095f980, 0xc4203ac060, 0x12, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:361 +0xb5b
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.2.1(0x7f4ce5286040, 0xc421070d50)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:297 +0x86
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4201dd480, 0xc42096e780, 0xc421070b70)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68799 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000063)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64db0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202617c0, 0xc42096e780, 0xc4203ad340)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68762 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc421b0f620)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203372f0, 0xc42096e780, 0xc4203372e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69051 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000056)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e1650)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340920, 0xc421a4c280, 0xc4201e8dc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69031 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4221e0f60)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331980, 0xc421a4c280, 0xc420331970)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68997 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000063)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38180)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203809d0, 0xc421a4c280, 0xc420214c20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68773 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000088)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64ba0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261590, 0xc42096e780, 0xc4203acf80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68792 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000005c)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e64cc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202616d0, 0xc42096e780, 0xc4203ad260)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69529 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc420bba900)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1116 +0x912
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:281 +0x1954

goroutine 69557 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func3(0x30cea60, 0xc4211d6300, 0xc421583200, 0xc4218e2160)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:236 +0x52e
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:256 +0x1389

goroutine 69526 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4206c6000, 0x7f4ce5286040, 0xc420b24c90, 0x14bf546b91a0108f, 0x0, 0x200000002, 0x2, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2253 +0x101d
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4206c6000, 0x7f4ce5286040, 0xc420b24c90, 0x14bf546b91a0108f, 0x0, 0x200000002, 0x2, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2091 +0xa9
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4206c6000, 0x7f4ce5286040, 0xc420b24c90, 0x14bf546b91a0108f, 0x0, 0x200000002, 0x2, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1425 +0x28e
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc420450800, 0x7f4ce5286040, 0xc420b24c60, 0x14bf546b91a0108f, 0x0, 0x200000002, 0x2, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2555 +0x9e0
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc420276080, 0x7f4ce5286040, 0xc420b24c30, 0x0, 0x0, 0x200000002, 0x2, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x24b
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x7f4ce5286040, 0xc420b24c30, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:843 +0x20f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc42096e780, 0x7f4ce5286040, 0xc420b24c00, 0xc4209d56a8, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:272 +0x14f
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc42106a000, 0x7f4ce5286040, 0xc420b24c00, 0xc421b84b60, 0xc420b24c00, 0x0, 0x6ac3c4)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:854 +0x1c1
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc42106a000, 0x7f4ce5286040, 0xc420b24c00, 0xc421b84b60, 0xc42106a000, 0x7f4ce52d52b0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:871 +0xb8
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x22123e0, 0xc42106a000, 0x7f4ce5286040, 0xc420b24b10, 0xc421b84a80, 0x0, 0x0, 0x0, 0x15, 0xc4212a3c98)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1877 +0x35b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4203355e0, 0x30ceac0, 0xc42167a0e0, 0xc420934240, 0xc4223d2bd0, 0x308e2d0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:738 +0xf68
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4203355e0, 0x30ceac0, 0xc42167a0e0, 0xc420934240, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:932 +0x122b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42020f5b0, 0xc4203355e0, 0x30ceac0, 0xc42167a0e0, 0xc420934240)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:497 +0xb7
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:498 +0xb9

goroutine 68330 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc421485290)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420294350, 0xc42096e780, 0xc4202f45a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68920 [chan receive]:
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc420353080, 0x28, 0xc4221f08d0, 0x7f4ce5316850, 0x0)
	<autogenerated>:9 +0x9f
github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith(0xc42017e078, 0x7f4ce5286040, 0xc420700f00, 0xc421a4c280, 0x30c6900, 0xc420353080, 0xc4221f08d0, 0xc420700f00, 0xc4208c7ea8)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:121 +0x61
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func9(0x7f4ce5286040, 0xc420700f00)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:622 +0x28f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420295450, 0xc421a4c280, 0xc421d14a80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69014 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc422908ae0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331690, 0xc421a4c280, 0xc420331680)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68334 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc420de73b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420294390, 0xc42096e780, 0xc4202f4680)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68952 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000044)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e1590)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203407e0, 0xc421a4c280, 0xc4201e8ae0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69269 [chan receive]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x7f4ce5286040, 0xc420b3e7e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:386 +0x94
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020f240, 0xc42056f5e0, 0xc420253600)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68852 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc420fde180)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1116 +0x912
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:281 +0x1954

goroutine 68828 [semacquire]:
sync.runtime_Semacquire(0xc420526504)
	/usr/local/go/src/runtime/sema.go:47 +0x34
sync.(*WaitGroup).Wait(0xc4205264f8)
	/usr/local/go/src/sync/waitgroup.go:131 +0xc2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc420526480, 0x7f4ce5286040, 0xc420e65410)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x3f
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait-fm(0x7f4ce5286040, 0xc420e65410)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3469 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261a80, 0xc42096e780, 0xc420261a70)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69020 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc422908c00)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203317f0, 0xc421a4c280, 0xc4203317e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68812 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000070)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e650b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202618d0, 0xc42096e780, 0xc4203ad580)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69587 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func3(0x30cea60, 0xc422886600, 0xc42137ab40, 0xc420e76420)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:236 +0x52e
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:256 +0x1389

goroutine 69075 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000071)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a384e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380b60, 0xc421a4c280, 0xc420214ec0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68875 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c360)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337540, 0xc42096e780, 0xc420337530)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69091 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1(0x7f4ce5286040, 0xc420e65c50)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3486 +0x481
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380c90, 0xc421a4c280, 0xc420380c80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68656 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).printStatsLoop(0xc420c8e270, 0x7f4ce5286040, 0xc420ec4180)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:253 +0xfce
github.com/cockroachdb/cockroach/pkg/kv.NewTxnCoordSender.func1(0x7f4ce5286040, 0xc420ec4180)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:237 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420249120, 0xc42096e780, 0xc420249110)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69037 [IO wait]:
net.runtime_pollWait(0x7f4ce653c5c8, 0x72, 0x30adc98)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc421d0bb88, 0x72, 0x30adc98, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc421d0bb88, 0xffffffffffffffff, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).accept(0xc421d0bb20, 0x0, 0x30b54c0, 0xc4201f3e40)
	/usr/local/go/src/net/fd_unix.go:430 +0x2c0
net.(*TCPListener).accept(0xc42017e080, 0xc420ad9e38, 0x6a695c, 0xc420ad9e10)
	/usr/local/go/src/net/tcpsock_posix.go:136 +0x51
net.(*TCPListener).Accept(0xc42017e080, 0x23112f8, 0xc421d14a00, 0x30d0980, 0xc42000e440)
	/usr/local/go/src/net/tcpsock.go:228 +0x50
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc421d14a00, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0xdd
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func10.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:741 +0x47
sync.(*Once).Do(0xc420295390, 0xc421956750)
	/usr/local/go/src/sync/once.go:44 +0xe2
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func10(0x7f4ce5286040, 0xc421a38810)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:742 +0xc3
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42022f3b0, 0xc421a4c280, 0xc4202813c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68885 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c4b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337690, 0xc42096e780, 0xc420337680)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69090 [semacquire]:
sync.runtime_Semacquire(0xc420fe0234)
	/usr/local/go/src/runtime/sema.go:47 +0x34
sync.(*WaitGroup).Wait(0xc420fe0228)
	/usr/local/go/src/sync/waitgroup.go:131 +0xc2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc420fe01b0, 0x7f4ce5286040, 0xc420e65bf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:174 +0x3f
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait-fm(0x7f4ce5286040, 0xc420e65bf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3469 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380c70, 0xc421a4c280, 0xc420380c60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69006 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000006c)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a383c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380aa0, 0xc421a4c280, 0xc420214de0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69035 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*Executor).Start.func1(0x7f4ce5286040, 0xc4221e1080)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:366 +0x19d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331a00, 0xc421a4c280, 0xc420346e80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68948 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000041)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e14a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420340780, 0xc421a4c280, 0xc4201e8a40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69033 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4221e1020)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203319d0, 0xc421a4c280, 0xc4203319c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68673 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1(0x7f4ce5286040, 0xc4214a6630)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:702 +0x2bd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203370e0, 0xc42096e780, 0xc4201d1ae0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69032 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4221e0ff0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203319a0, 0xc421a4c280, 0xc420331990)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69312 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc4211fd040)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:859 +0x7cf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc4211fd040)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:604 +0x267
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:605 +0xa4d

goroutine 69025 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4221e0e40)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203318a0, 0xc421a4c280, 0xc420331890)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69084 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000079)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a386c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380c00, 0xc421a4c280, 0xc420215060)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69019 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4203ba300)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203317d0, 0xc421a4c280, 0xc4203317c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69093 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func2(0x7f4ce5286040, 0xc420e65b30)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1222 +0x19d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380cc0, 0xc421a4c280, 0xc420215140)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68950 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000042)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e14d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203407b0, 0xc421a4c280, 0xc4201e8aa0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68886 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c510)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203376d0, 0xc42096e780, 0xc4203376c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69130 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420fe0120, 0xc421159180, 0xc420353f00, 0x12, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:361 +0xb5b
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.2.1(0x7f4ce5286040, 0xc4221c2ae0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:297 +0x86
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42019dbb0, 0xc421a4c280, 0xc4221c2a50)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69013 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc422908a50)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331670, 0xc421a4c280, 0xc420331660)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68869 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc420196b70)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337450, 0xc42096e780, 0xc420337440)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68826 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc40000007e)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420e653b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261a50, 0xc42096e780, 0xc4203ad7c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68873 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c300)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337500, 0xc42096e780, 0xc4203374f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69027 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc4221e0ea0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203318e0, 0xc421a4c280, 0xc4203318d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68368 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420d6f040)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:859 +0x7cf
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc420d6f040)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:604 +0x267
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:605 +0xa4d

goroutine 69082 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000077)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a38660)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380bd0, 0xc421a4c280, 0xc420215000)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68905 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc420e65d40)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ae780, 0xc421a4c280, 0xc4202be4a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68895 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).printStatsLoop(0xc420b125b0, 0x7f4ce5286040, 0xc421182ab0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:253 +0xfce
github.com/cockroachdb/cockroach/pkg/kv.NewTxnCoordSender.func1(0x7f4ce5286040, 0xc421182ab0)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:237 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420294940, 0xc421a4c280, 0xc420294930)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68872 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c270)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203374d0, 0xc42096e780, 0xc4203374c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69490 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc4205c9830, 0xc42177ce40, 0xc420316100, 0x12, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:361 +0xb5b
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.2.1(0x7f4ce5286040, 0xc4228bc120)
	/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:297 +0x86
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420394830, 0xc42096e780, 0xc4228bc0c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68918 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x7f4ce5286040, 0xc420700f30)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:547 +0xd2
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420295430, 0xc421a4c280, 0xc421d14a40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69215 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x7f4ce5286040, 0xc421a38e40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:102 +0x8e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020f030, 0xc42056f5e0, 0xc420253240)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68984 [select]:
github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1(0x7f4ce5286040, 0xc420b3fad0)
	/go/src/github.com/cockroachdb/cockroach/pkg/ts/db.go:90 +0x1c7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331340, 0xc421a4c280, 0xc420331330)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68914 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x7f4ce5286040, 0xc420700cf0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:102 +0x8e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202952d0, 0xc421a4c280, 0xc420352e60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 57402 [select, locked to thread]:
runtime.gopark(0x2312cd8, 0x0, 0x2242e64, 0x6, 0x18, 0x2)
	/usr/local/go/src/runtime/proc.go:271 +0xfd
runtime.selectgoImpl(0xc420b7cf50, 0xc420b7cf38, 0x0)
	/usr/local/go/src/runtime/select.go:423 +0x138a
runtime.selectgo(0xc420b7cf50)
	/usr/local/go/src/runtime/select.go:238 +0x1c
runtime.ensureSigM.func1()
	/usr/local/go/src/runtime/signal_unix.go:434 +0x1ea
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1

goroutine 68951 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000043)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e1500)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203407d0, 0xc421a4c280, 0xc4201e8ac0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68798 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000062)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420de71a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202617b0, 0xc42096e780, 0xc4203ad320)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69038 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc42095a400, 0x7f4ce5286000, 0xc421bcf280, 0x1dcd6500, 0x0, 0xc42095a448, 0x1, 0xc420379040, 0x3, 0x3, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1237 +0x1cef
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc42095a400, 0x7f4ce5286000, 0xc421bcf280, 0x1, 0xc420379040, 0x3, 0x3, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:405 +0x3b4
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc42095a400, 0x7f4ce5286040, 0xc4205765d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420956d00, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:474 +0x1a8
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc42095a400, 0x7f4ce5286040, 0xc4205765d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420956d00, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:940 +0x4ec
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc42095a400, 0x7f4ce5286040, 0xc4205765d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420956d00, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:810 +0xb78
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc42095a400, 0x7f4ce5286040, 0xc4205765d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420956b60, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:624 +0x3ce
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc420b125b0, 0x7f4ce5286040, 0xc4205764e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420956a90, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:478 +0x490
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc420352640, 0x7f4ce5286000, 0xc4209cdf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420956a90, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:528 +0x228
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).send(0xc420e77a20, 0x7f4ce5286000, 0xc4209cdf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:864 +0x59b
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x7f4ce5286000, 0xc4209cdf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:434 +0x97
github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x7f4ce5286000, 0xc4209cdf00, 0xc42067f638, 0xc420e71100, 0xc4216eb570, 0xc42067f760)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:436 +0x13b
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc420e77a20, 0x7f4ce5286000, 0xc4209cdf00, 0xc420e71100, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:434 +0x112
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).updateLiveness.func1(0x7f4ce5286000, 0xc4209cdf00, 0xc420e77a20, 0xc42067f7d8, 0xbf41cc)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:545 +0x599
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Txn.func1(0x7f4ce5286000, 0xc4209cdf00, 0xc420e77a20, 0xc42020fb63, 0xc42067f8f8, 0x6ac6f2)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:490 +0x5f
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Exec(0xc420e77a20, 0x7f4ce5286000, 0xc4209cdf00, 0xc420010101, 0xc42020fb80, 0x28, 0xc420120810)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:640 +0xe3
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Txn(0xc420352640, 0x7f4ce5286000, 0xc4209cdf00, 0xc420214500, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x17a
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).updateLiveness(0xc420120780, 0x7f4ce5286040, 0xc420b3fe90, 0xc420b3ff20, 0xc420b3fec0, 0xc420791bb0, 0x3535700, 0x1000000006d0f01)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:546 +0x18d
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).heartbeatInternal.func1(0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:333 +0x4e5
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).heartbeatInternal(0xc420120780, 0x7f4ce5286040, 0xc420b3fe90, 0xc420b3fec0, 0x3ff8000000000000, 0x0, 0x3fc3333333333333)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:349 +0x130
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1(0x7f4ce5286040, 0xc420b258f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:248 +0x52e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42022f3d0, 0xc421a4c280, 0xc420c1d4a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68331 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc420de7320)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420294360, 0xc42096e780, 0xc4202f45c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69058 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc40000005c)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e1830)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203409b0, 0xc421a4c280, 0xc4201e8ee0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69530 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc420bba900)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1162 +0x6e6
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:283 +0x1a4b

goroutine 68716 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000082)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420de6f00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202614f0, 0xc42096e780, 0xc4203ace20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68779 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000051)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421485080)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261600, 0xc42096e780, 0xc4203ad060)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68949 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000040)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc4221e1470)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203407a0, 0xc421a4c280, 0xc4201e8a80)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69480 [IO wait]:
net.runtime_pollWait(0x7f4ce653cc88, 0x72, 0x6fb68a)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc4218beca8, 0x72, 0x30adc98, 0xffffffffffffffff)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc4218beca8, 0xc4228c0000, 0x8000)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).Read(0xc4218bec40, 0xc4228c0000, 0x8000, 0x8000, 0x0, 0x30b8100, 0x30adc98)
	/usr/local/go/src/net/fd_unix.go:250 +0x22b
net.(*conn).Read(0xc420816368, 0xc4228c0000, 0x8000, 0x8000, 0x69fb14, 0xc42273aa98, 0x6cbed0)
	/usr/local/go/src/net/net.go:181 +0x97
net.(*TCPConn).Read(0xc420816368, 0xc4228c0000, 0x8000, 0x8000, 0xc42001e000, 0x67318c, 0xc4208f8300)
	<autogenerated>:72 +0x73
bufio.(*Reader).Read(0xc4228be2a0, 0xc420ce0e38, 0x9, 0x9, 0x6ef873, 0xc4223b61a0, 0xffffffff)
	/usr/local/go/src/bufio/bufio.go:213 +0x7e7
io.ReadAtLeast(0x30af940, 0xc4228be2a0, 0xc420ce0e38, 0x9, 0x9, 0x9, 0x6d0f42, 0x6ac60c, 0xc422862d20)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30af940, 0xc4228be2a0, 0xc420ce0e38, 0x9, 0x9, 0xc4228e7d10, 0xc4228e7d10, 0xc42273ae98)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420ce0e38, 0x9, 0x9, 0x30af940, 0xc4228be2a0, 0x0, 0x3700000000, 0x37, 0x37)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420ce0e00, 0x30ba4c0, 0xc4228e7ce0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x118
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc4228b4060, 0xc4228e7ce0, 0xc4228e7c00, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc422886600)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1043 +0x15b
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:251 +0x1294

goroutine 68770 [semacquire]:
sync.runtime_notifyListWait(0xc421d141d0, 0xc400000085)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d141c0)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420526480, 0xc42096e780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc420de6fc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261560, 0xc42096e780, 0xc4203acf20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68888 [select]:
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).Start.func1(0x7f4ce5286040, 0xc4206aa9c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:115 +0x61c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331640, 0xc42096e780, 0xc420331630)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68830 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startCoalescedHeartbeatsLoop.func1(0x7f4ce5286040, 0xc420de71d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3517 +0x1c5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420261ad0, 0xc42096e780, 0xc420261ab0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68850 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc421587440, 0x7f4ce5286040, 0xc420e65590, 0xed0adf3d7, 0x259b3c92, 0x3535700, 0xc420c230c0, 0xc42096e780, 0xc4206c6000, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:210 +0x282
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc4206c6000, 0x58)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:272 +0x162
github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc42048a7b0, 0xc4211b1d60)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:358 +0x5ad
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1(0x7f4ce5286040, 0xc420e65590)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x3f0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202943e0, 0xc42096e780, 0xc4202f4700)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69256 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc42081b200)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1116 +0x912
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:281 +0x1954

goroutine 69602 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease.func2(0xc420a3bf88, 0xc421844f00, 0xc420507880, 0x7f4ce5286040, 0xc4223d3b60, 0xc4227dea10, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1075 +0x72c
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).redirectOnOrAcquireLease(0xc420507880, 0x7f4ce5286040, 0xc4223d3b60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1125 +0x25d
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc420507880, 0x7f4ce5286040, 0xc4223d3b60, 0x14bf546baf8ad1aa, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2204 +0x1298
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc420507880, 0x7f4ce5286040, 0xc4223d3b60, 0x14bf546baf8ad1aa, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2091 +0xa9
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc420507880, 0x7f4ce5286040, 0xc4223d3b60, 0x14bf546baf8ad1aa, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1425 +0x28e
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4204d6800, 0x7f4ce5286040, 0xc4223d3b30, 0x14bf546baf8ad1aa, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2555 +0x9e0
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42007a700, 0x7f4ce5286040, 0xc4223d3b00, 0x0, 0x0, 0x300000003, 0x3, 0x1, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x24b
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x7f4ce5286040, 0xc4223d3b00, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:843 +0x20f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421a4c280, 0x7f4ce5286040, 0xc4223d3ad0, 0xc420a3d6a8, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:272 +0x14f
github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc4200b4000, 0x7f4ce5286040, 0xc4223d3ad0, 0xc4227de930, 0xc4223d3ad0, 0x0, 0x6ac3c4)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:854 +0x1c1
github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc4200b4000, 0x7f4ce5286040, 0xc4223d3ad0, 0xc4227de930, 0xc4200b4000, 0x7f4ce5294e10, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/node.go:871 +0xb8
github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x22123e0, 0xc4200b4000, 0x7f4ce5286040, 0xc4223d39e0, 0xc4227de8c0, 0x0, 0x0, 0x0, 0x15, 0xc4208f0c98)
	/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:1877 +0x35b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4200a41c0, 0x30ceac0, 0xc420b742a0, 0xc420d25560, 0xc420e262d0, 0x308e2d0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:738 +0xf68
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4200a41c0, 0x30ceac0, 0xc420b742a0, 0xc420d25560, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:932 +0x122b
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420365b90, 0xc4200a41c0, 0x30ceac0, 0xc420b742a0, 0xc420d25560)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:497 +0xb7
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:498 +0xb9

goroutine 69074 [semacquire]:
sync.runtime_notifyListWait(0xc421d14e50, 0xc400000070)
	/usr/local/go/src/runtime/sema.go:298 +0x11a
sync.(*Cond).Wait(0xc421d14e40)
	/usr/local/go/src/sync/cond.go:57 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420fe01b0, 0xc421a4c280)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:199 +0xe2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x7f4ce5286040, 0xc421a384b0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:168 +0x4b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420380b50, 0xc421a4c280, 0xc420214ea0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68769 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc420196990)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203373d0, 0xc42096e780, 0xc4203373c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69483 [IO wait]:
net.runtime_pollWait(0x7f4ce653cb08, 0x72, 0x6fb68a)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc42285d108, 0x72, 0x30adc98, 0xffffffffffffffff)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc42285d108, 0xc4228de000, 0x8000)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).Read(0xc42285d0a0, 0xc4228de000, 0x8000, 0x8000, 0x0, 0x30b8100, 0x30adc98)
	/usr/local/go/src/net/fd_unix.go:250 +0x22b
net.(*conn).Read(0xc42000e440, 0xc4228de000, 0x8000, 0x8000, 0x208, 0x0, 0x2312bc8)
	/usr/local/go/src/net/net.go:181 +0x97
net.(*TCPConn).Read(0xc42000e440, 0xc4228de000, 0x8000, 0x8000, 0x1, 0x30b0040, 0xc420016110)
	<autogenerated>:72 +0x73
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc420d23ae0, 0xc4228de000, 0x8000, 0x8000, 0x7f4cc7562b48, 0xc421187910, 0x67ee91)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x183
bufio.(*Reader).Read(0xc4228be3c0, 0xc420ce0ff8, 0x9, 0x9, 0x7f4cc7562b48, 0xc4228e7208, 0x0)
	/usr/local/go/src/bufio/bufio.go:213 +0x7e7
io.ReadAtLeast(0x30af940, 0xc4228be3c0, 0xc420ce0ff8, 0x9, 0x9, 0x9, 0xc421187a48, 0x6ef873, 0xc4206b41a0)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30af940, 0xc4228be3c0, 0xc420ce0ff8, 0x9, 0x9, 0x0, 0x7f4ce661be10, 0x6d0f42)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420ce0ff8, 0x9, 0x9, 0x30af940, 0xc4228be3c0, 0x0, 0xc400000000, 0xc, 0x1ff)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420ce0fc0, 0x30ba4c0, 0xc4210a1230, 0x0, 0xc420807800)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x118
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc4228b41b0, 0xc4210a1230, 0xc4210a1230, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420ce1180, 0xc4228b4300, 0x2312020)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:307 +0x3c4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4200a41c0, 0x30ceac0, 0xc420ce1180)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:505 +0x1e8
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4200a41c0, 0x30d0860, 0xc420d23ae0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:486 +0x5a6
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4200a41c0, 0x30d0860, 0xc420d23ae0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:457 +0x5f4
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:426 +0x69c

goroutine 68902 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x7f4ce5286040, 0xc420e65e90)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:490 +0x468
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4202ae750, 0xc421a4c280, 0xc4202be420)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69268 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func9(0x7f4ce5286040, 0xc421a38ff0)
	/go/src/github.com/cockroachdb/cockroach/pkg/server/server.go:607 +0x397
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42020f150, 0xc42056f5e0, 0xc4209cce40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 69193 [IO wait]:
net.runtime_pollWait(0x7f4ce52cd8b8, 0x72, 0x6fb68a)
	/usr/local/go/src/runtime/netpoll.go:164 +0x5e
net.(*pollDesc).wait(0xc4200a7e98, 0x72, 0x30adc98, 0xffffffffffffffff)
	/usr/local/go/src/net/fd_poll_runtime.go:75 +0x5b
net.(*pollDesc).waitRead(0xc4200a7e98, 0xc421cbe000, 0x8000)
	/usr/local/go/src/net/fd_poll_runtime.go:80 +0x42
net.(*netFD).Read(0xc4200a7e30, 0xc421cbe000, 0x8000, 0x8000, 0x0, 0x30b8100, 0x30adc98)
	/usr/local/go/src/net/fd_unix.go:250 +0x22b
net.(*conn).Read(0xc42017e1c0, 0xc421cbe000, 0x8000, 0x8000, 0x21823c0, 0x0, 0x2312bc8)
	/usr/local/go/src/net/net.go:181 +0x97
net.(*TCPConn).Read(0xc42017e1c0, 0xc421cbe000, 0x8000, 0x8000, 0x1, 0x30b0040, 0xc420016110)
	<autogenerated>:72 +0x73
github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42107c820, 0xc421cbe000, 0x8000, 0x8000, 0x1d0ecc0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x183
bufio.(*Reader).Read(0xc420d06ba0, 0xc420b74118, 0x9, 0x9, 0x7f4cd380dec8, 0xc4223d3928, 0x0)
	/usr/local/go/src/bufio/bufio.go:213 +0x7e7
io.ReadAtLeast(0x30af940, 0xc420d06ba0, 0xc420b74118, 0x9, 0x9, 0x9, 0xc4215dda48, 0x6ef873, 0xc4220b9520)
	/usr/local/go/src/io/io.go:307 +0xb7
io.ReadFull(0x30af940, 0xc420d06ba0, 0xc420b74118, 0x9, 0x9, 0xc4203af860, 0x0, 0x1)
	/usr/local/go/src/io/io.go:325 +0x73
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420b74118, 0x9, 0x9, 0x30af940, 0xc420d06ba0, 0x0, 0xc400000000, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xa3
github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420b740e0, 0x0, 0x30b0040, 0xc420016110, 0xc42284bdc0)
	/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0x118
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc42050ffb0, 0xc4223d3a10, 0xc4223d3a10, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http_util.go:508 +0x53
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420b742a0, 0xc420ac6180, 0x2312020)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:307 +0x3c4
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4200a41c0, 0x30ceac0, 0xc420b742a0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:505 +0x1e8
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport(0xc4200a41c0, 0x30d0860, 0xc42107c820, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:486 +0x5a6
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn(0xc4200a41c0, 0x30d0860, 0xc42107c820)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:457 +0x5f4
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:426 +0x69c

goroutine 68871 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc42038c210)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203374b0, 0xc42096e780, 0xc4203374a0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68870 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*SchemaChangeManager).Start.func1(0x7f4ce5286040, 0xc420196c00)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/schema_changer.go:807 +0x1f0c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420337460, 0xc42096e780, 0xc4201d1c40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb

goroutine 68989 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*distSQLPlanner).initRunners.func1(0x7f4ce5286040, 0xc422908840)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:83 +0x1cb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420331480, 0xc421a4c280, 0xc420331460)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:215 +0x161
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:216 +0xbb


ERROR: exit status 255

make: *** [stress] Error 1
25 runs completed, 1 failures, over 11m31s
FAIL
Makefile:224: recipe for target 'stress' failed
