Project: usergrid
Issue USERGRID-755: MVCC serialization error on reindex
2015-06-16 22:06:31,223 [http-bio-8080-exec-15] ERROR org.apache.usergrid.persistence.collection.serialization.impl.MvccLogEntrySerializationStrategyImpl- DATA CORRUPTION DETECTED when de-serializing entity with Id SimpleId{uuid=2034290e-109c-11e5-a3f5-06b0ee5388b2, type='user'}. This means the write was truncated. org.apache.usergrid.persistence.collection.exception.DataCorruptionException: Unable to read entity data org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl$EntitySerializer.fromByteBuffer(MvccEntitySerializationStrategyV3Impl.java:457) org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl$EntitySerializer.fromByteBuffer(MvccEntitySerializationStrategyV3Impl.java:365) com.netflix.astyanax.serializers.AbstractSerializer.fromBytes(AbstractSerializer.java:42) com.netflix.astyanax.thrift.model.ThriftColumnImpl.getValue(ThriftColumnImpl.java:58) org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl$MvccColumnParser.parseColumn(MvccEntitySerializationStrategyV3Impl.java:346) org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl.lambda$load$13(MvccEntitySerializationStrategyV3Impl.java:210) org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl$$Lambda$23/1238550786.call(Unknown Source) rx.Observable$5.call(Observable.java:3534) rx.internal.operators.OperatorScan$2.onNext(OperatorScan.java:104) rx.observers.SerializedObserver.onNext(SerializedObserver.java:159) rx.observers.SerializedSubscriber.onNext(SerializedSubscriber.java:95) rx.internal.operators.NotificationLite.accept(NotificationLite.java:150) rx.internal.operators.OperatorMergeMaxConcurrent$SourceSubscriber.drain(OperatorMergeMaxConcurrent.java:265) rx.internal.operators.OperatorMergeMaxConcurrent$SourceSubscriber$MergeItemSubscriber.onNext(OperatorMergeMaxConcurrent.java:325) rx.internal.operators.OperatorSubscribeOn$1$1$1.onNext(OperatorSubscribeOn.java:76) rx.internal.operators.OperatorMap$1.onNext(OperatorMap.java:55) rx.internal.util.ScalarSynchronousObservable$1.call(ScalarSynchronousObservable.java:43) rx.internal.util.ScalarSynchronousObservable$1.call(ScalarSynchronousObservable.java:32) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable.unsafeSubscribe(Observable.java:7495) rx.internal.operators.OperatorSubscribeOn$1$1.call(OperatorSubscribeOn.java:62) rx.schedulers.ImmediateScheduler$InnerImmediateScheduler.schedule(ImmediateScheduler.java:58) rx.internal.operators.OperatorSubscribeOn$1.onNext(OperatorSubscribeOn.java:57) rx.internal.operators.OperatorSubscribeOn$1.onNext(OperatorSubscribeOn.java:43) rx.internal.util.ScalarSynchronousObservable$1.call(ScalarSynchronousObservable.java:43) rx.internal.util.ScalarSynchronousObservable$1.call(ScalarSynchronousObservable.java:32) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable.unsafeSubscribe(Observable.java:7495) rx.internal.operators.OperatorMergeMaxConcurrent$SourceSubscriber.subscribeNext(OperatorMergeMaxConcurrent.java:147) rx.internal.operators.OperatorMergeMaxConcurrent$SourceSubscriber.onNext(OperatorMergeMaxConcurrent.java:126) rx.internal.operators.OperatorMergeMaxConcurrent$SourceSubscriber.onNext(OperatorMergeMaxConcurrent.java:68) rx.internal.operators.OperatorMap$1.onNext(OperatorMap.java:55) rx.internal.operators.OperatorBufferWithSize$1.onNext(OperatorBufferWithSize.java:103) rx.internal.operators.OnSubscribeFromIterable$IterableProducer.request(OnSubscribeFromIterable.java:96) rx.internal.operators.OperatorBufferWithSize$1$1.request(OperatorBufferWithSize.java:88) rx.Subscriber.setProducer(Subscriber.java:177) rx.Subscriber.setProducer(Subscriber.java:171) rx.internal.operators.OperatorBufferWithSize$1.setProducer(OperatorBufferWithSize.java:74) rx.internal.operators.OnSubscribeFromIterable.call(OnSubscribeFromIterable.java:47) rx.internal.operators.OnSubscribeFromIterable.call(OnSubscribeFromIterable.java:33) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable.subscribe(Observable.java:7585) rx.observables.BlockingObservable.blockForSingle(BlockingObservable.java:442) rx.observables.BlockingObservable.last(BlockingObservable.java:229) org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl.load(MvccEntitySerializationStrategyV3Impl.java:215) org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyProxyImpl.load(MvccEntitySerializationStrategyProxyImpl.java:95) org.apache.usergrid.persistence.collection.impl.EntityCollectionManagerImpl$1.call(EntityCollectionManagerImpl.java:246) org.apache.usergrid.persistence.collection.impl.EntityCollectionManagerImpl$1.call(EntityCollectionManagerImpl.java:240) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable$1.call(Observable.java:144) rx.Observable$1.call(Observable.java:136) rx.Observable.subscribe(Observable.java:7585) rx.observables.BlockingObservable.blockForSingle(BlockingObservable.java:442) rx.observables.BlockingObservable.lastOrDefault(BlockingObservable.java:262) org.apache.usergrid.corepersistence.CpEntityManager.load(CpEntityManager.java:316) org.apache.usergrid.corepersistence.CpEntityManager.get(CpEntityManager.java:454) org.apache.usergrid.corepersistence.CpEntityManager.get(CpEntityManager.java:2344) org.apache.usergrid.management.cassandra.ManagementServiceImpl.getUserEntityByIdentifier(ManagementServiceImpl.java:1060) org.apache.usergrid.management.cassandra.ManagementServiceImpl.findUserEntity(ManagementServiceImpl.java:1132) org.apache.usergrid.management.cassandra.ManagementServiceImpl.verifyAdminUserPasswordCredentials(ManagementServiceImpl.java:1277) org.apache.usergrid.rest.management.ManagementResource.getAccessTokenInternal(ManagementResource.java:256) org.apache.usergrid.rest.management.ManagementResource.getAccessTokenPostJson(ManagementResource.java:396) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:483) com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$ResponseOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:205) com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1542) com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1473) com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419) com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409) com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409) com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:540) com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:909) com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:857) com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:811) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) org.apache.shiro.web.servlet.AbstractShiroFilter.executeChain(AbstractShiroFilter.java:449) org.apache.shiro.web.servlet.AbstractShiroFilter$1.call(AbstractShiroFilter.java:365) org.apache.shiro.subject.support.SubjectCallable.doCall(SubjectCallable.java:90) org.apache.shiro.subject.support.SubjectCallable.call(SubjectCallable.java:83) org.apache.shiro.subject.support.DelegatingSubject.execute(DelegatingSubject.java:383) org.apache.shiro.web.servlet.AbstractShiroFilter.doFilterInternal(AbstractShiroFilter.java:362) org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125) org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:343) org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:260) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) org.apache.usergrid.rest.filters.ContentTypeFilter.doFilter(ContentTypeFilter.java:92) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220) org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122) org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:504) org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:170) org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103) org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950) org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116) org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:421) org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1074) org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:611) org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) java.lang.Thread.run(Thread.java:745) Caused by: java.io.CharConversionException: Invalid UTF-32 character 0x7d000000(above 10ffff) at char #1, byte #7) com.fasterxml.jackson.core.io.UTF32Reader.reportInvalid(UTF32Reader.java:155) com.fasterxml.jackson.core.io.UTF32Reader.read(UTF32Reader.java:109) com.fasterxml.jackson.core.json.ReaderBasedJsonParser.loadMore(ReaderBasedJsonParser.java:131) com.fasterxml.jackson.core.json.ReaderBasedJsonParser._skipWSOrEnd(ReaderBasedJsonParser.java:1688) com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:562) com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:3031) com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:2978) com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:2167) org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyV3Impl$EntitySerializer.fromByteBuffer(MvccEntitySerializationStrategyV3Impl.java:446)	... 145 more
Story points: 3