Repository: LangStream/langstream (language: python)

Issue:
Forbidden paths gives null pointer exception when processing a URL with no path
I have configured the web crawler to scrape a number website and to also include some forbidden paths, like this:

```
17:22:06.905 [main] INFO  a.l.a.webcrawler.WebCrawlerSource -- allowed-domains: [https://langstream.ai/, https://gi
thub.com/LangStream/langstream, https://docs.langstream.ai/]
17:22:06.905 [main] INFO  a.l.a.webcrawler.WebCrawlerSource -- forbidden-paths: [/LangStream/langstream/pulls, /LangStream/langstream/pull, /LangStream/langstream/commits, /LangStream/langstream/pulse, /LangStream/langstream/issues, /LangStream/langstream/actions, /LangStream/langstream/commit]
17:22:06.905 [main] INFO  a.l.a.webcrawler.WebCrawlerSource -- seed-urls: [https://github.com/LangStream/langstream/, https://langstream.ai/, https://docs.langstream.ai/]
```

When the crawler tries to process `langstream.ai`, which has no path, it crashes with a null pointer exception:

```
17:22:39.810 [main] INFO  a.l.a.w.crawler.WebCrawlerStatus -- PendingUrls: 26 Uncommitted 26
17:22:39.810 [main] INFO  a.l.a.webcrawler.crawler.WebCrawler -- Crawling url: https://langstream.ai/
17:22:39.988 [main] INFO  o.a.k.clients.producer.KafkaProducer -- [Producer clientId=producer-1] Closing the Kafka
producer with timeoutMillis = 9223372036854775807 ms.
17:22:39.995 [main] INFO  o.a.kafka.common.metrics.Metrics -- Metrics scheduler closed
...
17:22:40.195 [main] INFO  a.l.runtime.agent.AgentRunnerStarter -- Error, NOW SLEEPING
java.lang.NullPointerException: Cannot invoke "String.startsWith(String)" because "pathOnly" is null
        at ai.langstream.agents.webcrawler.crawler.WebCrawlerConfiguration.lambda$isAllowedUrl$1(WebCrawlerConfiguration.java:80)
        at java.base/java.util.stream.MatchOps$1MatchSink.accept(MatchOps.java:90)
        at java.base/java.util.HashMap$KeySpliterator.tryAdvance(HashMap.java:1728)
        at java.base/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129)
        at java.base/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527)
        at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513)
        at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
        at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:230)
        at java.base/java.util.stream.MatchOps$MatchOp.evaluateSequential(MatchOps.java:196)
        at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
        at java.base/java.util.stream.ReferencePipeline.anyMatch(ReferencePipeline.java:632)
        at ai.langstream.agents.webcrawler.crawler.WebCrawlerConfiguration.isAllowedUrl(WebCrawlerConfiguration.java:76)
        at ai.langstream.agents.webcrawler.crawler.WebCrawler.isUrlForbidden(WebCrawler.java:283)
        at ai.langstream.agents.webcrawler.crawler.WebCrawler.lambda$runCycle$0(WebCrawler.java:231)
        at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
        at ai.langstream.agents.webcrawler.crawler.WebCrawler.runCycle(WebCrawler.java:227)
        at ai.langstream.agents.webcrawler.WebCrawlerSource.read(WebCrawlerSource.java:224)
        at ai.langstream.api.runner.code.AgentCodeAndLoader$1.lambda$read$0(AgentCodeAndLoader.java:87)
        at ai.langstream.api.runner.code.AgentCodeAndLoader.callWithContextClassloader(AgentCodeAndLoader.java:66)
        at ai.langstream.api.runner.code.AgentCodeAndLoader$1.read(AgentCodeAndLoader.java:87)
        at ai.langstream.runtime.agent.AgentRunner$PendingRecordsCounterSource.read(AgentRunner.java:499)
        at ai.langstream.runtime.agent.AgentRunner.runMainLoop(AgentRunner.java:641)
        at ai.langstream.runtime.agent.AgentRunner.runJavaAgent(AgentRunner.java:412)
        at ai.langstream.runtime.agent.AgentRunner.run(AgentRunner.java:190)
        at ai.langstream.runtime.agent.AgentRunnerStarter.start(AgentRunnerStarter.java:111)
        at ai.langstream.runtime.agent.AgentRunnerStarter.main(AgentRunnerStarter.java:53)
        at ai.langstream.runtime.Main.main(Main.java:42)
```


Discussion / hints:
The problem appears to be that `getPath()` returns null here in `WebCrawlerConfiguration.java`:

```
        try {
            URI uri = URI.create(url);
            String host = uri.getHost();
            pathOnly = uri.getPath();
            domainOnly = host;
        } catch (Exception e) {
            log.info("Url {} doesn't have a domain, parsing error: {}", url, e + "");
            return false;
        }
```

Fix patch:
diff --git a/examples/applications/docker-chatbot/chatbot.yaml b/examples/applications/docker-chatbot/chatbot.yaml
index f408844f6..51783f6ea 100644
--- a/examples/applications/docker-chatbot/chatbot.yaml
+++ b/examples/applications/docker-chatbot/chatbot.yaml
@@ -88,7 +88,7 @@ pipeline:
               Do not provide information that is not related to the LangStream project.
             
               Documents:
-              {{ value.related_documents}}
+              {{# value.related_documents}}
               {{ text}}
               {{/ value.related_documents}}
         - role: user
diff --git a/examples/applications/query-milvus/chatbot.yaml b/examples/applications/query-milvus/chatbot.yaml
index e77a3ca2b..1872ac1ea 100644
--- a/examples/applications/query-milvus/chatbot.yaml
+++ b/examples/applications/query-milvus/chatbot.yaml
@@ -80,7 +80,7 @@ pipeline:
               Do not provide information that is not related to the LangStream project.
             
               Documents:
-              {{ value.related_documents}}
+              {{# value.related_documents}}
               {{ text}}
               {{/ value.related_documents}}
         - role: user
diff --git a/examples/applications/webcrawler-source/chatbot.yaml b/examples/applications/webcrawler-source/chatbot.yaml
index fc4fe8526..7a51b973b 100644
--- a/examples/applications/webcrawler-source/chatbot.yaml
+++ b/examples/applications/webcrawler-source/chatbot.yaml
@@ -87,7 +87,7 @@ pipeline:
               Do not provide information that is not related to the LangStream project.
             
               Documents:
-              {{ value.related_documents}}
+              {{# value.related_documents}}
               {{ text}}
               {{/ value.related_documents}}
         - role: user
diff --git a/langstream-agents/langstream-agent-webcrawler/src/main/java/ai/langstream/agents/webcrawler/crawler/WebCrawlerConfiguration.java b/langstream-agents/langstream-agent-webcrawler/src/main/java/ai/langstream/agents/webcrawler/crawler/WebCrawlerConfiguration.java
index 9998c766f..bea4e5b41 100644
--- a/langstream-agents/langstream-agent-webcrawler/src/main/java/ai/langstream/agents/webcrawler/crawler/WebCrawlerConfiguration.java
+++ b/langstream-agents/langstream-agent-webcrawler/src/main/java/ai/langstream/agents/webcrawler/crawler/WebCrawlerConfiguration.java
@@ -44,11 +44,23 @@ public class WebCrawlerConfiguration {
 
     public boolean isAllowedUrl(String url) {
         final String domainOnly;
-        final String pathOnly;
+        final String path;
         try {
+            final String pathOnly;
             URI uri = URI.create(url);
             String host = uri.getHost();
             pathOnly = uri.getPath();
+            if (pathOnly == null) {
+                // this doesn't look like a real URL
+                // for instance it is something like http:something (without slashes)
+                return false;
+            }
+            log.info("pathOnly: {}", pathOnly);
+            if (pathOnly.isEmpty()) {
+                path = "/";
+            } else {
+                path = pathOnly;
+            }
             domainOnly = host;
         } catch (Exception e) {
             log.info("Url {} doesn't have a domain, parsing error: {}", url, e + "");
@@ -74,10 +86,10 @@ public boolean isAllowedUrl(String url) {
                 !forbiddenPaths.isEmpty()
                         && forbiddenPaths.stream()
                                 .anyMatch(
-                                        path -> {
+                                        fpath -> {
                                             // https://domain/something/....
                                             // http://domain/....
-                                            return pathOnly.startsWith(path);
+                                            return path.startsWith(fpath);
                                         });
 
         return allowedDomain && !forbiddenPath;


Test patch:
diff --git a/langstream-agents/langstream-agent-webcrawler/src/test/java/ai/langstream/agents/webcrawler/crawler/WebCrawlerConfigurationTest.java b/langstream-agents/langstream-agent-webcrawler/src/test/java/ai/langstream/agents/webcrawler/crawler/WebCrawlerConfigurationTest.java
index 019ed8c23..9a89ab2db 100644
--- a/langstream-agents/langstream-agent-webcrawler/src/test/java/ai/langstream/agents/webcrawler/crawler/WebCrawlerConfigurationTest.java
+++ b/langstream-agents/langstream-agent-webcrawler/src/test/java/ai/langstream/agents/webcrawler/crawler/WebCrawlerConfigurationTest.java
@@ -53,6 +53,14 @@ void testForbiddenPaths() {
                         "https://domain/something/secondlevel", Set.of("/something/sec")));
 
         assertFalse(verifyForbiddenPaths("not-an-url", Set.of("/something")));
+
+        // this is not an URL
+        assertFalse(verifyForbiddenPaths("something:somewhere", Set.of("/something")));
+
+        assertTrue(verifyForbiddenPaths("https://domain", Set.of("/something")));
+        assertTrue(verifyForbiddenPaths("https://domain/", Set.of("/something")));
+        assertFalse(verifyForbiddenPaths("https://domain", Set.of("/")));
+        assertFalse(verifyForbiddenPaths("https://domain/", Set.of("/")));
     }
 
     private boolean verifyDomain(String url, Set<String> allowedDomains) {