Project: springxd
Issue XD-3181: XD Container not closing file descriptors
Currently when running the {{p-spring-xd}} stream tests, we run into an issue where the XD Container starts throwing errors because it cannot open the module configuration file. This happens reliably after 3-4 days of running and always fails on the same {{modules.yml}} configuration file. This is not to say that the file leak is guaranteed to be related to the {{modules.yml}}, but it's certainly a place to start looking. A restart of the container (only) causes the error to go away for 3-4 days at which point it reappears, indicating that the problem is definitely in the XD Container and not in the operating system's configuration. Failure stack trace: {noformat} [XD ADMIN] 1.2.0.RC1 INFO DeploymentSupervisor-0 zk.ZKStreamDeploymentHandler - Deployment status for stream 'end-to-end-http-9630': DeploymentStatus{state=failed,error(s)=java.lang.IllegalStateException: java.io.FileNotFoundException: /var/vcap/jobs/xd-container/packages/spring-xd/config/modules/modules.yml (Too many open files) at org.springframework.beans.factory.config.YamlProcessor.handleProcessError(YamlProcessor.java:186) at org.springframework.beans.factory.config.YamlProcessor.process(YamlProcessor.java:178) at org.springframework.beans.factory.config.YamlProcessor.process(YamlProcessor.java:138) at org.springframework.boot.env.YamlPropertySourceLoader$Processor.process(YamlPropertySourceLoader.java:100) at org.springframework.boot.env.YamlPropertySourceLoader.load(YamlPropertySourceLoader.java:57) at org.springframework.boot.env.PropertySourcesLoader.load(PropertySourcesLoader.java:126) at org.springframework.boot.context.config.ConfigFileApplicationListener$Loader.loadIntoGroup(ConfigFileApplicationListener.java:381) at org.springframework.boot.context.config.ConfigFileApplicationListener$Loader.load(ConfigFileApplicationListener.java:369) at org.springframework.boot.context.config.ConfigFileApplicationListener$Loader.load(ConfigFileApplicationListener.java:339) at org.springframework.boot.context.config.ConfigFileApplicationListener.addPropertySources(ConfigFileApplicationListener.java:174) at org.springframework.xd.module.options.EnvironmentAwareModuleOptionsMetadataResolver$1.apply(EnvironmentAwareModuleOptionsMetadataResolver.java:229) at org.springframework.xd.module.options.EnvironmentAwareModuleOptionsMetadataResolver.loadPropertySources(EnvironmentAwareModuleOptionsMetadataResolver.java:219) at org.springframework.xd.module.options.EnvironmentAwareModuleOptionsMetadataResolver.lookupEnvironment(EnvironmentAwareModuleOptionsMetadataResolver.java:181) at org.springframework.xd.module.options.EnvironmentAwareModuleOptionsMetadataResolver.access$000(EnvironmentAwareModuleOptionsMetadataResolver.java:61) at org.springframework.xd.module.options.EnvironmentAwareModuleOptionsMetadataResolver$ModuleOptionsMetadataWithDefaults.(EnvironmentAwareModuleOptionsMetadataResolver.java:144) at org.springframework.xd.module.options.EnvironmentAwareModuleOptionsMetadataResolver.resolve(EnvironmentAwareModuleOptionsMetadataResolver.java:132) at org.springframework.xd.dirt.stream.XDStreamParser.buildModuleDescriptors(XDStreamParser.java:206) at org.springframework.xd.dirt.stream.XDStreamParser.parse(XDStreamParser.java:122) at org.springframework.xd.dirt.stream.StreamFactory.createStream(StreamFactory.java:84) at org.springframework.xd.dirt.server.admin.deployment.zk.DeploymentLoader.loadStream(DeploymentLoader.java:101) at org.springframework.xd.dirt.server.container.DeploymentListener.deployStreamModule(DeploymentListener.java:331) at org.springframework.xd.dirt.server.container.DeploymentListener.onChildAdded(DeploymentListener.java:181) at org.springframework.xd.dirt.server.container.DeploymentListener.childEvent(DeploymentListener.java:149) at org.apache.curator.framework.recipes.cache.PathChildrenCache$5.apply(PathChildrenCache.java:509) at org.apache.curator.framework.recipes.cache.PathChildrenCache$5.apply(PathChildrenCache.java:503) at org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:92) at com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) at org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:83) at org.apache.curator.framework.recipes.cache.PathChildrenCache.callListeners(PathChildrenCache.java:500) at org.apache.curator.framework.recipes.cache.EventOperation.invoke(EventOperation.java:35) at org.apache.curator.framework.recipes.cache.PathChildrenCache$10.run(PathChildrenCache.java:762) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.FileNotFoundException: /var/vcap/jobs/xd-container/packages/spring-xd/config/modules/modules.yml (Too many open files) at java.io.FileInputStream.open0(Native Method) at java.io.FileInputStream.open(FileInputStream.java:195) at java.io.FileInputStream.(FileInputStream.java:138) at java.io.FileInputStream.(FileInputStream.java:93) at sun.net.www.protocol.file.FileURLConnection.connect(FileURLConnection.java:90) at sun.net.www.protocol.file.FileURLConnection.getInputStream(FileURLConnection.java:188) at org.springframework.core.io.UrlResource.getInputStream(UrlResource.java:168) at org.springframework.beans.factory.config.YamlProcessor.process(YamlProcessor.java:158) ... 36 more ; java.lang.IllegalStateException: java.io.FileNotFoundException: /var/vcap/jobs/xd-container/packages/spring-xd/config/modules/modules.yml (Too many open files) at org.springframework.beans.factory.config.YamlProcessor.handleProcessError(YamlProcessor.java:186) at org.springframework.beans.factory.config.YamlProcessor.process(YamlProcessor.java:178) at org.springframework.beans.factory.config.YamlProcessor.process(YamlProcessor.java:138) at org.springframework.boot.env.YamlPropertySourceLoader$Processor.process(YamlPropertySourceLoader.java:100) at org.springframework.boot.env.YamlPropertySourceLoader.load(YamlPropertySourceLoader.java:57) at org.springframework.boot.env.PropertySourcesLoader.load(PropertySourcesLoader.java:126) at org.springframework.boot.context.config.ConfigFileApplicationListener$Loader.loadIntoGroup(ConfigFileApplicationListener.java:381) at org.springframework.boot.context.config.ConfigFileApplicationListener$Loader.load(ConfigFileApplicationListener.java:369) at org.springframework.boot.context.config.ConfigFileApplicationListener$Loader.load(ConfigFileApplicationListener.java:339) at org.springframework.boot.context.config.ConfigFileApplicationListener.addPropertySources(ConfigFileApplicationListener.java:174) at org.springframework.xd.module.options.EnvironmentAwareModuleOptionsMetadataResolver$1.apply(EnvironmentAwareModuleOptionsMetadataResolver.java:229) at org.springframework.xd.module.options.EnvironmentAwareModuleOptionsMetadataResolver.loadPropertySources(EnvironmentAwareModuleOptionsMetadataResolver.java:219) at org.springframework.xd.module.options.EnvironmentAwareModuleOptionsMetadataResolver.lookupEnvironment(EnvironmentAwareModuleOptionsMetadataResolver.java:181) at org.springframework.xd.module.options.EnvironmentAwareModuleOptionsMetadataResolver.access$000(EnvironmentAwareModuleOptionsMetadataResolver.java:61) at org.springframework.xd.module.options.EnvironmentAwareModuleOptionsMetadataResolver$ModuleOptionsMetadataWithDefaults.(EnvironmentAwareModuleOptionsMetadataResolver.java:144) at org.springframework.xd.module.options.EnvironmentAwareModuleOptionsMetadataResolver.resolve(EnvironmentAwareModuleOptionsMetadataResolver.java:132) at org.springframework.xd.dirt.stream.XDStreamParser.buildModuleDescriptors(XDStreamParser.java:206) at org.springframework.xd.dirt.stream.XDStreamParser.parse(XDStreamParser.java:122) at org.springframework.xd.dirt.stream.StreamFactory.createStream(StreamFactory.java:84) at org.springframework.xd.dirt.server.admin.deployment.zk.DeploymentLoader.loadStream(DeploymentLoader.java:101) at org.springframework.xd.dirt.server.container.DeploymentListener.deployStreamModule(DeploymentListener.java:331) at org.springframework.xd.dirt.server.container.DeploymentListener.onChildAdded(DeploymentListener.java:181) at org.springframework.xd.dirt.server.container.DeploymentListener.childEvent(DeploymentListener.java:149) at org.apache.curator.framework.recipes.cache.PathChildrenCache$5.apply(PathChildrenCache.java:509) at org.apache.curator.framework.recipes.cache.PathChildrenCache$5.apply(PathChildrenCache.java:503) at org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:92) at com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) at org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:83) at org.apache.curator.framework.recipes.cache.PathChildrenCache.callListeners(PathChildrenCache.java:500) at org.apache.curator.framework.recipes.cache.EventOperation.invoke(EventOperation.java:35) at org.apache.curator.framework.recipes.cache.PathChildrenCache$10.run(PathChildrenCache.java:762) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.FileNotFoundException: /var/vcap/jobs/xd-container/packages/spring-xd/config/modules/modules.yml (Too many open files) at java.io.FileInputStream.open0(Native Method) at java.io.FileInputStream.open(FileInputStream.java:195) at java.io.FileInputStream.(FileInputStream.java:138) at java.io.FileInputStream.(FileInputStream.java:93) at sun.net.www.protocol.file.FileURLConnection.connect(FileURLConnection.java:90) at sun.net.www.protocol.file.FileURLConnection.getInputStream(FileURLConnection.java:188) at org.springframework.core.io.UrlResource.getInputStream(UrlResource.java:168) at org.springframework.beans.factory.config.YamlProcessor.process(YamlProcessor.java:158) ... 36 more } {noformat}
Story points: 8