init

.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

.gitignore

@@ -0,0 +1,36 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+.yarn/install-state.gz
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

Dockerfile

@@ -0,0 +1,62 @@
+FROM node:18 AS base
+
+# Install dependencies only when needed
+FROM base AS deps
+
+WORKDIR /app
+
+# Install dependencies based on the preferred package manager
+COPY package.json yarn.lock* package-lock.json* pnpm-lock.yaml* ./
+RUN \
+  if [ -f yarn.lock ]; then yarn --frozen-lockfile; \
+  elif [ -f package-lock.json ]; then npm ci; \
+  elif [ -f pnpm-lock.yaml ]; then yarn global add pnpm && pnpm i --frozen-lockfile; \
+  else echo "Lockfile not found." && exit 1; \
+  fi
+
+# Uncomment the following lines if you want to use a secret at buildtime, 
+# for example to access your private npm packages
+# RUN --mount=type=secret,id=HF_EXAMPLE_SECRET,mode=0444,required=true \
+#     $(cat /run/secrets/HF_EXAMPLE_SECRET)
+
+# Rebuild the source code only when needed
+FROM base AS builder
+WORKDIR /app
+COPY --from=deps /app/node_modules ./node_modules
+COPY . .
+
+# Next.js collects completely anonymous telemetry data about general usage.
+# Learn more here: https://nextjs.org/telemetry
+# Uncomment the following line in case you want to disable telemetry during the build.
+# ENV NEXT_TELEMETRY_DISABLED 1
+
+# RUN yarn build
+
+# If you use yarn, comment out this line and use the line above
+RUN npm run build
+
+# Production image, copy all the files and run next
+FROM base AS runner
+WORKDIR /app
+
+ENV NODE_ENV production
+# Uncomment the following line in case you want to disable telemetry during runtime.
+# ENV NEXT_TELEMETRY_DISABLED 1
+
+RUN addgroup --system --gid 1001 nodejs
+RUN adduser --system --uid 1001 nextjs
+
+COPY --from=builder /app/public ./public
+
+# Automatically leverage output traces to reduce image size
+# https://nextjs.org/docs/advanced-features/output-file-tracing
+COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
+COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
+
+USER nextjs
+
+EXPOSE 3000
+
+ENV PORT 3000
+
+CMD ["node", "server.js"]

README.md

@@ -1,10 +1,23 @@
 ---
-title: Url Surfer
-emoji: 🌍
-colorFrom: blue
-colorTo: red
+title: url-surfer
+emoji: 🔗🏄‍♂️
 sdk: docker
-pinned: false
+app_port: 3000
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🔗 url-surfer 🏄‍♂️
+
+Simple API to navigate to a URL from a prompt, returning relevant context from the prompt using a vector store.
+
+## How it works
+
+1. Navigate (parse url/fetch/mime-type check)
+2. Extract text (jsdom/puppeteer/pdf-parse)
+3. Create vector store (transformers.js)
+4. Return vector results in prompt (langchain)
+
+
+## Ideas
+
+* Configuration: Vector search size, token counting
+* OpenAI functions integration

app/api/chat/route.ts

@@ -0,0 +1,29 @@
+import { Configuration, OpenAIApi } from "openai-edge";
+import { OpenAIStream, StreamingTextResponse } from "ai";
+import { createUrlSurfer } from "@/app/tools/surfer";
+
+const [, requestSchema] = createUrlSurfer();
+
+const config = new Configuration({
+  apiKey: process.env.OPENAI_API_KEY,
+});
+const openai = new OpenAIApi(config);
+
+const functions: any[] = [
+  requestSchema
+];
+
+export async function POST(req: Request) {
+  const { messages, function_call } = await req.json()
+
+  const response = await openai.createChatCompletion({
+    model: 'gpt-4',
+    stream: true,
+    messages,
+    functions,
+    function_call
+  })
+
+  const stream = OpenAIStream(response)
+  return new StreamingTextResponse(stream)
+}

app/globals.css

@@ -0,0 +1,107 @@
+:root {
+  --max-width: 1100px;
+  --border-radius: 12px;
+  --font-mono: ui-monospace, Menlo, Monaco, 'Cascadia Mono', 'Segoe UI Mono',
+    'Roboto Mono', 'Oxygen Mono', 'Ubuntu Monospace', 'Source Code Pro',
+    'Fira Mono', 'Droid Sans Mono', 'Courier New', monospace;
+
+  --foreground-rgb: 0, 0, 0;
+  --background-start-rgb: 214, 219, 220;
+  --background-end-rgb: 255, 255, 255;
+
+  --primary-glow: conic-gradient(
+    from 180deg at 50% 50%,
+    #16abff33 0deg,
+    #0885ff33 55deg,
+    #54d6ff33 120deg,
+    #0071ff33 160deg,
+    transparent 360deg
+  );
+  --secondary-glow: radial-gradient(
+    rgba(255, 255, 255, 1),
+    rgba(255, 255, 255, 0)
+  );
+
+  --tile-start-rgb: 239, 245, 249;
+  --tile-end-rgb: 228, 232, 233;
+  --tile-border: conic-gradient(
+    #00000080,
+    #00000040,
+    #00000030,
+    #00000020,
+    #00000010,
+    #00000010,
+    #00000080
+  );
+
+  --callout-rgb: 238, 240, 241;
+  --callout-border-rgb: 172, 175, 176;
+  --card-rgb: 180, 185, 188;
+  --card-border-rgb: 131, 134, 135;
+}
+
+@media (prefers-color-scheme: dark) {
+  :root {
+    --foreground-rgb: 255, 255, 255;
+    --background-start-rgb: 0, 0, 0;
+    --background-end-rgb: 0, 0, 0;
+
+    --primary-glow: radial-gradient(rgba(1, 65, 255, 0.4), rgba(1, 65, 255, 0));
+    --secondary-glow: linear-gradient(
+      to bottom right,
+      rgba(1, 65, 255, 0),
+      rgba(1, 65, 255, 0),
+      rgba(1, 65, 255, 0.3)
+    );
+
+    --tile-start-rgb: 2, 13, 46;
+    --tile-end-rgb: 2, 5, 19;
+    --tile-border: conic-gradient(
+      #ffffff80,
+      #ffffff40,
+      #ffffff30,
+      #ffffff20,
+      #ffffff10,
+      #ffffff10,
+      #ffffff80
+    );
+
+    --callout-rgb: 20, 20, 20;
+    --callout-border-rgb: 108, 108, 108;
+    --card-rgb: 100, 100, 100;
+    --card-border-rgb: 200, 200, 200;
+  }
+}
+
+* {
+  box-sizing: border-box;
+  padding: 0;
+  margin: 0;
+}
+
+html,
+body {
+  max-width: 100vw;
+  overflow-x: hidden;
+}
+
+body {
+  color: rgb(var(--foreground-rgb));
+  background: linear-gradient(
+      to bottom,
+      transparent,
+      rgb(var(--background-end-rgb))
+    )
+    rgb(var(--background-start-rgb));
+}
+
+a {
+  color: inherit;
+  text-decoration: none;
+}
+
+@media (prefers-color-scheme: dark) {
+  html {
+    color-scheme: dark;
+  }
+}

app/icons.tsx

@@ -0,0 +1,21 @@
+
+export const FunctionIcon = ({ className }: { className?: string }) => {
+  return (
+    <svg
+      xmlns="http://www.w3.org/2000/svg"
+      fill="none"
+      viewBox="0 0 24 24"
+      width={25}
+      height={25}
+      strokeWidth={1.8}
+      stroke="currentColor"
+      className={className}
+    >
+      <path
+        strokeLinecap="round"
+        strokeLinejoin="round"
+        d="M4.745 3A23.933 23.933 0 003 12c0 3.183.62 6.22 1.745 9M19.5 3c.967 2.78 1.5 5.817 1.5 9s-.533 6.22-1.5 9M8.25 8.885l1.444-.89a.75.75 0 011.105.402l2.402 7.206a.75.75 0 001.104.401l1.445-.889m-8.25.75l.213.09a1.687 1.687 0 002.062-.617l4.45-6.676a1.688 1.688 0 012.062-.618l.213.09"
+      />
+    </svg>
+  );
+};

app/layout.tsx

@@ -0,0 +1,22 @@
+import type { Metadata } from 'next'
+import { Inter } from 'next/font/google'
+import './globals.css'
+
+const inter = Inter({ subsets: ['latin'] })
+
+export const metadata: Metadata = {
+  title: '🔗 URL Surfer 🏄‍♂️',
+  description: 'Navigate to URLs and perform realtime similarity search',
+}
+
+export default function RootLayout({
+  children,
+}: {
+  children: React.ReactNode
+}) {
+  return (
+    <html lang="en">
+      <body className={inter.className}>{children}</body>
+    </html>
+  )
+}

app/page.module.css

@@ -0,0 +1,415 @@
+
+
+.main {
+  display: flex;
+  flex-direction: column;
+  justify-content: space-between;
+  padding: 6rem;
+}
+
+.description {
+  display: inherit;
+  justify-content: inherit;
+  align-items: inherit;
+  font-size: 0.85rem;
+  max-width: var(--max-width);
+  width: 100%;
+  z-index: 2;
+  font-family: var(--font-mono);
+}
+
+.description a {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  gap: 0.5rem;
+}
+
+.description p {
+  position: relative;
+  margin: 0;
+  padding: 1rem;
+  background-color: rgba(var(--callout-rgb), 0.5);
+  border: 1px solid rgba(var(--callout-border-rgb), 0.3);
+  border-radius: var(--border-radius);
+}
+
+.code {
+  font-weight: 700;
+  font-family: var(--font-mono);
+}
+
+.grid {
+  display: grid;
+  grid-template-columns: repeat(4, minmax(25%, auto));
+  max-width: 100%;
+  width: var(--max-width);
+}
+
+.card {
+  padding: 1rem 1.2rem;
+  border-radius: var(--border-radius);
+  background: rgba(var(--card-rgb), 0);
+  border: 1px solid rgba(var(--card-border-rgb), 0);
+  transition: background 200ms, border 200ms;
+}
+
+.card span {
+  display: inline-block;
+  transition: transform 200ms;
+}
+
+.card h2 {
+  font-weight: 600;
+  margin-bottom: 0.7rem;
+}
+
+.card p {
+  margin: 0;
+  opacity: 0.6;
+  font-size: 0.9rem;
+  line-height: 1.5;
+  max-width: 30ch;
+}
+
+.center {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  position: relative;
+  padding: 4rem 0;
+}
+
+.center::before {
+  background: var(--secondary-glow);
+  border-radius: 50%;
+  width: 480px;
+  height: 360px;
+  margin-left: -400px;
+}
+
+.center::after {
+  background: var(--primary-glow);
+  width: 240px;
+  height: 180px;
+  z-index: -1;
+}
+
+.center::before,
+.center::after {
+  content: '';
+  left: 50%;
+  position: absolute;
+  filter: blur(45px);
+  transform: translateZ(0);
+}
+
+.logo {
+  position: relative;
+}
+/* Enable hover only on non-touch devices */
+@media (hover: hover) and (pointer: fine) {
+  .card:hover {
+    background: rgba(var(--card-rgb), 0.1);
+    border: 1px solid rgba(var(--card-border-rgb), 0.15);
+  }
+
+  .card:hover span {
+    transform: translateX(4px);
+  }
+}
+
+@media (prefers-reduced-motion) {
+  .card:hover span {
+    transform: none;
+  }
+}
+
+/* Mobile */
+@media (max-width: 700px) {
+  .content {
+    padding: 4rem;
+  }
+
+  .grid {
+    grid-template-columns: 1fr;
+    margin-bottom: 120px;
+    max-width: 320px;
+    text-align: center;
+  }
+
+  .card {
+    padding: 1rem 2.5rem;
+  }
+
+  .card h2 {
+    margin-bottom: 0.5rem;
+  }
+
+  .center {
+    padding: 8rem 0 6rem;
+  }
+
+  .center::before {
+    transform: none;
+    height: 300px;
+  }
+
+  .description {
+    font-size: 0.8rem;
+  }
+
+  .description a {
+    padding: 1rem;
+  }
+
+  .description p,
+  .description div {
+    display: flex;
+    justify-content: center;
+    position: fixed;
+    width: 100%;
+  }
+
+  .description p {
+    align-items: center;
+    inset: 0 0 auto;
+    padding: 2rem 1rem 1.4rem;
+    border-radius: 0;
+    border: none;
+    border-bottom: 1px solid rgba(var(--callout-border-rgb), 0.25);
+    background: linear-gradient(
+      to bottom,
+      rgba(var(--background-start-rgb), 1),
+      rgba(var(--callout-rgb), 0.5)
+    );
+    background-clip: padding-box;
+    backdrop-filter: blur(24px);
+  }
+
+  .description div {
+    align-items: flex-end;
+    pointer-events: none;
+    inset: auto 0 0;
+    padding: 2rem;
+    height: 200px;
+    background: linear-gradient(
+      to bottom,
+      transparent 0%,
+      rgb(var(--background-end-rgb)) 40%
+    );
+    z-index: 1;
+  }
+}
+
+/* Tablet and Smaller Desktop */
+@media (min-width: 701px) and (max-width: 1120px) {
+  .grid {
+    grid-template-columns: repeat(2, 50%);
+  }
+}
+
+@media (prefers-color-scheme: dark) {
+  .vercelLogo {
+    filter: invert(1);
+  }
+
+  .logo {
+    filter: invert(1) drop-shadow(0 0 0.3rem #ffffff70);
+  }
+}
+
+@keyframes rotate {
+  from {
+    transform: rotate(360deg);
+  }
+  to {
+    transform: rotate(0deg);
+  }
+}
+
+.response pre {
+  white-space: pre-wrap;
+  word-break: break-word;
+  overflow-wrap: break-word;
+  overflow-y: auto;
+  max-height: 500px; /* Maximum height before scrolling */
+}
+
+.input {
+  padding: 0.5rem 1rem; /* Adjust the padding inside the input */
+  margin: 0.5rem 0; /* Adds margin around the input for spacing */
+  border: 1px solid #ccc; /* A light border for the input */
+  border-radius: 4px; /* Rounded corners */
+  font-size: 1rem; /* Base font size */
+  line-height: 1.5; /* Height of the input line */
+  color: #333; /* Text color */
+  background-color: #fff; /* Background color of the input */
+  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); /* Optional: Adds a subtle shadow */
+  width: 100%; /* Make input full width of the parent container */
+  box-sizing: border-box; /* Include padding and border in the element's total width and height */
+  transition: border-color 0.3s, box-shadow 0.3s; /* Smooth transition for focus effect */
+}
+
+.input:focus {
+  border-color: #007bff; /* Highlight color when input is focused */
+  outline: none; /* Removes the default focus outline */
+  box-shadow: 0 0 0 2px rgba(0, 123, 255, 0.25); /* Adds a glow effect on focus */
+}
+
+.button {
+  padding: 0.75rem 1.5rem; /* Adjust padding to increase the button's size */
+  margin-top: 1rem; /* Space above the button */
+  border: none; /* No border for a button */
+  border-radius: 25px; /* Rounded corners for a pill shape */
+  font-size: 1rem; /* Font size */
+  font-weight: bold; /* Make the text a bit bolder */
+  color: white; /* Text color */
+  background-image: linear-gradient(to right, #6ee7b7, #3b82f6); /* Gradient background */
+  cursor: pointer; /* Change cursor to indicate it's clickable */
+  transition: transform 0.2s, background-color 0.3s; /* Smooth transitions for interactions */
+  text-transform: uppercase; /* Optional: uppercase text */
+  letter-spacing: 0.05em; /* Spacing between letters */
+  box-shadow: 0 4px 6px rgba(50, 50, 93, 0.11), 0 1px 3px rgba(0, 0, 0, 0.08); /* Subtle shadow for depth */
+}
+
+.button:hover, .button:focus {
+  background-image: linear-gradient(to left, #6ee7b7, #3b82f6); /* Change the direction of gradient on hover/focus */
+  transform: translateY(-2px); /* Slightly raise the button */
+  box-shadow: 0 7px 14px rgba(50, 50, 93, 0.1), 0 3px 6px rgba(0, 0, 0, 0.08); /* Increase shadow size */
+}
+
+.button:active {
+  transform: translateY(0); /* Button goes back down when clicked */
+  box-shadow: 0 3px 6px rgba(50, 50, 93, 0.16), 0 2px 4px rgba(0, 0, 0, 0.1); /* Smaller shadow when button is pressed */
+}
+
+.button:disabled {
+  background-image: linear-gradient(to right, #cbd5e1, #94a3b8); /* Less vibrant gradient for disabled state */
+  cursor: default; /* No pointer cursor since it's not clickable */
+  box-shadow: none; /* No shadow for a flat appearance */
+}
+.main {
+  display: flex;
+  flex-direction: column;
+}
+
+.title {
+  color: #333;
+  text-align: center;
+  margin-bottom: 2rem;
+}
+
+.messages {
+  flex: 1;
+  min-height: 80vh;
+}
+
+/* Chat message styles */
+.message {
+  background-color: #f9f9f9;
+  border-radius: 15px;
+  padding: 10px 15px;
+  margin: 8px 0;
+  max-width: 80%;
+  word-break: break-word; /* Ensures text breaks nicely */
+}
+
+/* Different background for messages from the 'user' role */
+.message-user {
+  background-color: #dcf8c6 !important;
+}
+
+/* Avatar styles */
+.avatar {
+  width: 40px;
+  height: 40px;
+  border-radius: 50%;
+  background-color: #ccc;
+  margin-right: 10px;
+}
+
+/* Style for function call messages */
+.function-call {
+  color: #555;
+  font-style: italic;
+}
+
+/* Input field styles */
+.input {
+  padding: 10px;
+  margin: 10px 0;
+  width: 100%;
+  box-sizing: border-box;
+  border: 1px solid #ddd;
+  border-radius: 4px;
+}
+
+/* Button styles */
+.button {
+  padding: 10px 20px;
+  background-color: #5cb85c;
+  color: white;
+  border: none;
+  border-radius: 4px;
+  cursor: pointer;
+}
+
+.button:disabled {
+  background-color: #ccc;
+}
+
+/* Form styles */
+.form {
+  width: 100%;
+  max-width: 500px;
+}
+
+.flex {
+  display: flex;
+  align-items: flex-start;
+}
+
+.text-gray-500 {
+  color: #737373;
+}
+
+.font-bold {
+  font-weight: bold;
+}
+
+/* Additional media query for responsive design */
+@media (max-width: 640px) {
+  .message {
+    max-width: 100%;
+  }
+}
+
+/* In your CSS file */
+.avatar-user {
+  width: 20px; /* Based on Tailwind's width setting */
+}
+
+.bg-white {
+  background-color: white;
+}
+
+.bg-black {
+  background-color: black;
+}
+
+.bg-gray-100 {
+  background-color: #f7fafc; /* Approximation of Tailwind's gray-100 */
+}
+
+.bg-green-500 {
+  background-color: #48bb78; /* Approximation of Tailwind's green-500 */
+}
+
+.bg-gray-200 {
+  background-color: #edf2f7; /* Approximation of Tailwind's gray-200 */
+}
+
+.bg-blue-500 {
+  background-color: #4299e1; /* Approximation of Tailwind's blue-500 */
+}

app/page.tsx

@@ -0,0 +1,183 @@
+"use client";
+
+import styles from './page.module.css';
+import { useState } from 'react';
+import { useChat } from 'ai/react';
+import { FunctionCallHandler, Message, nanoid } from 'ai';
+import ReactMarkdown from "react-markdown";
+import { Bot, User } from "lucide-react";
+import { toast } from 'sonner';
+import { FunctionIcon } from './icons';
+
+const Page: React.FC = () => {
+  const functionCallHandler: FunctionCallHandler = async (
+    chatMessages,
+    functionCall,
+  ) => {
+    let result;
+    const response = await fetch("/api/surfer", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        prompt: input
+      })
+    } as any);
+
+    if (!response.ok) {
+      const errorText = await response.text();
+      toast.error(`Something went wrong: ${errorText}`);
+      return;
+    }    
+
+    result = await response.text();
+
+    return {
+      messages: [
+        ...chatMessages,
+        {
+          id: nanoid(),
+          name: functionCall.name,
+          role: "function" as const,
+          content: result,
+        },
+      ],
+    };
+  };
+  const { messages, input, setInput, handleSubmit, isLoading } = useChat({
+    experimental_onFunctionCall: functionCallHandler,
+    onResponse: (response: { status: number; }) => {
+      if (response.status === 429) {
+        toast.error("You have reached your request limit for the day.");
+        return;
+      } else {
+        console.log("chat initialized");
+      }
+    },
+    onError: (error: any) => {
+      console.log(error);
+    },
+  });
+  const [isExpanded, setIsExpanded] = useState(false);
+  const toggleExpand = () => {
+    setIsExpanded(!isExpanded);
+  };
+
+  const roleUIConfig: {
+    [key: string]: {
+      avatar: JSX.Element;
+      bgColor: string;
+      avatarColor: string;
+      // eslint-disable-next-line no-unused-vars
+      dialogComponent: (message: Message) => JSX.Element;
+    };
+  } = {
+    user: {
+      avatar: <User width={20} />,
+      bgColor: "bg-white",
+      avatarColor: "bg-black",
+      dialogComponent: (message: Message) => (
+        <ReactMarkdown
+          className=""
+          components={{
+            a: (props) => (
+              <a {...props} target="_blank" rel="noopener noreferrer" />
+            ),
+          }}
+        >
+          {message.content}
+        </ReactMarkdown>
+      ),
+    },
+    assistant: {
+      avatar: <Bot width={20} />,
+      bgColor: "bg-gray-100",
+      avatarColor: "bg-green-500",
+      dialogComponent: (message: Message) => (
+        <ReactMarkdown
+          className=""
+          components={{
+            a: (props) => (
+              <a {...props} target="_blank" rel="noopener noreferrer" />
+            ),
+          }}
+        >
+          {message.content}
+        </ReactMarkdown>
+      ),
+    },
+    function: {
+      avatar: <div className="cursor-pointer" onClick={toggleExpand}><FunctionIcon /></div>,
+      bgColor: "bg-gray-200",
+      avatarColor: "bg-blue-500",
+      dialogComponent: (message: Message) => {
+        return (
+          <div className="flex flex-col">
+            {isExpanded && (
+              <div className="py-1">{message.content}</div>
+            )}
+          </div>
+        );
+      },
+    }    
+  };
+
+
+  return (
+    <main className={styles.main}>
+      <h1 className={styles.title}>
+        🔗 URL Surfer 🏄‍♂️
+      </h1>
+      <div className={styles.messages}>
+        {messages.length > 0 ? (
+          messages.map((message, i) => {
+            const messageClass = `message ${message.role === 'user' ? 'message-user' : ''}`;
+            return (
+              <div key={i} className={messageClass}>
+                <div className="flex w-full max-w-screen-md items-start space-x-4 px-5 sm:px-0">
+                  <div className="avatar">
+                    {roleUIConfig[message.role].avatar}
+                  </div>
+                  {message.content === "" && message.function_call != undefined ? (
+                    typeof message.function_call === "object" ? (
+                      <div className="flex flex-col">
+                        <div>
+                          Using{" "}
+                          <span className="font-bold">
+                            {message.function_call.name}
+                          </span>{" "}
+                          ...
+                        </div>
+                        <div className="">
+                          {message.function_call.arguments}
+                        </div>
+                      </div>
+                    ) : (
+                      <div className="function-call">{message.function_call}</div>
+                    )
+                  ) : (
+                    roleUIConfig[message.role].dialogComponent(message)
+                  )}
+                </div>
+              </div>
+            );
+        })) : null}
+      </div>
+      <form onSubmit={handleSubmit} className={styles.form}>
+        <input
+          type="text"
+          value={input}
+          onChange={(e) => setInput(e.target.value)}
+          placeholder="Enter URL and Prompt"
+          className={styles.input}
+        />
+        <button type="submit" className={styles.button} disabled={isLoading}>
+          {isLoading ? 'Loading...' : 'Submit'}
+        </button>
+      </form>
+    </main>
+  );
+};
+
+export default Page;

app/tools/surfer.ts

@@ -0,0 +1,36 @@
+import { Tool } from 'openai-function-calling-tools';
+import { z } from 'zod';
+
+function createUrlSurfer() {
+  const paramsSchema = z.object({
+    input: z.string(),
+  });
+  const name = 'surfer';
+  const description = 'A custom URL navigator. Useful when a URL is provided with a question. Input should be a prompt with a URL. Outputs a JSON array of relevant results.';
+
+  const execute = async ({ input }: z.infer<typeof paramsSchema>) => {
+    try {
+      const res = await fetch('/api/surfer', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ prompt: input }),
+      });
+
+      if (!res.ok) {
+        throw new Error(`HTTP error! status: ${res.status}`);
+      }
+
+      const data = await res.json();
+      return data;
+    } catch (error) {
+      // @ts-ignore
+      throw new Error(`Error in UrlSurfer: ${error.message}`);
+    }
+  };
+
+  return new Tool(paramsSchema, name, description, execute).tool;
+}
+
+export { createUrlSurfer };

next.config.js

@@ -0,0 +1,6 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+    output: "standalone"
+}
+
+module.exports = nextConfig

package.json

@@ -0,0 +1,47 @@
+{
+  "name": "url-surfer",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "@emotion/react": "^11.11.1",
+    "@emotion/styled": "^11.11.0",
+    "@mui/material": "^5.14.16",
+    "@upstash/ratelimit": "^0.4.4",
+    "@vercel/kv": "^0.2.4",
+    "@xenova/transformers": "^2.7.0",
+    "ai": "^2.2.20",
+    "jsdom": "^22.1.0",
+    "langchain": "^0.0.180",
+    "lucide-react": "^0.292.0",
+    "next": "14.0.1",
+    "node-fetch": "^3.3.2",
+    "openai": "^3.3.0",
+    "openai-edge": "^1.2.2",
+    "openai-function-calling-tools": "^6.1.3",
+    "pdf-parse": "^1.1.1",
+    "puppeteer": "^19.11.1",
+    "react": "^18",
+    "react-dom": "^18",
+    "react-markdown": "^9.0.0",
+    "sonner": "^1.1.0",
+    "zod": "^3.22.4",
+    "zod-to-json-schema": "^3.21.4"
+  },
+  "devDependencies": {
+    "@types/jsdom": "^21.1.4",
+    "@types/node": "^20",
+    "@types/pdf-parse": "^1.1.3",
+    "@types/puppeteer": "^7.0.4",
+    "@types/react": "^18",
+    "@types/react-dom": "^18",
+    "eslint": "^8",
+    "eslint-config-next": "14.0.1",
+    "typescript": "^5"
+  }
+}

pages/api/surfer/index.ts

@@ -0,0 +1,94 @@
+import { NextApiRequest, NextApiResponse } from 'next';
+import fetch from 'node-fetch';
+import { JSDOM } from 'jsdom';
+// @ts-ignore
+import pdfParse from 'pdf-parse';
+import puppeteer from 'puppeteer';
+import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
+import { MemoryVectorStore } from 'langchain/vectorstores/memory';
+import { HuggingFaceTransformersEmbeddings } from "langchain/embeddings/hf_transformers";
+
+export const config = {
+    api: {
+      bodyParser: {
+        sizeLimit: '1mb',
+      },
+    },
+};
+
+const DEFAULT_CHUNK_SIZE = 1000;
+const VECTOR_STORE_SIZE = 10;
+const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: DEFAULT_CHUNK_SIZE });
+
+async function extractTextFromPDF(buffer: Buffer): Promise<string> {
+  const data = await pdfParse(buffer);
+  return data.text;
+}
+
+const model = new HuggingFaceTransformersEmbeddings({
+    modelName: "Xenova/all-MiniLM-L6-v2",
+});
+
+const urlRegex = /(https?:\/\/[^\s]+)/g;
+
+const handleContentText = async (targetUrl: string) => {
+  const response = await fetch(targetUrl);
+  const contentType = response.headers.get('content-type') || '';
+  let content;
+  if (contentType.includes('application/pdf')) {
+    const buffer = await response.arrayBuffer();
+    content = await extractTextFromPDF(buffer as any);
+  } else if (contentType.includes('text/html')) {
+    const html = await response.text();
+    const dom = new JSDOM(html);
+    const scripts = dom.window.document.querySelectorAll('script, style');
+    scripts.forEach(element => element.remove());
+    content = dom.window.document.body.textContent || '';
+
+    if (!content.trim()) {
+      const browser = await puppeteer.launch();
+      const page = await browser.newPage();
+      await page.goto(targetUrl);
+      content = await page.evaluate(() => document.body.innerText);
+      await browser.close();
+    }
+  } else {
+    content = await response.text();
+  }
+  return content;
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  const prompt = req.body.prompt as string;
+  const urls = prompt.match(urlRegex);
+  const targetUrl = urls ? urls[0] : null;
+  const promptWithoutUrl = urls ? prompt.replace(urlRegex, '').trim() : prompt;
+
+  if (!targetUrl) {
+    return `Couldn't find url, here is the ${prompt}`;
+  }
+  
+  try {
+    const content: string = await handleContentText(targetUrl)
+    if (!content) {
+      return `Couldn't find ${targetUrl}, here is the prompt: ${promptWithoutUrl}`;
+    }
+    
+    const documents = await textSplitter.createDocuments([content]);
+
+    const vectorStore = await MemoryVectorStore.fromTexts(
+      // @ts-ignore
+      [...documents.map(doc => doc.pageContent)],
+      // @ts-ignore
+      [...documents.map((v, k) => k)],
+      model
+    )
+    const queryResult = await vectorStore.similaritySearch(promptWithoutUrl, VECTOR_STORE_SIZE);
+    return res.status(200).send(
+      `Here is the context: ${JSON.stringify(queryResult.map(result => result.pageContent))} from using the prompt to lookup relevant information. Here is the prompt: ${promptWithoutUrl}`);
+  } catch (error) {
+    console.error(error);
+    // @ts-ignore
+    return res.status(500).json({ error: error.message });
+  }
+}

tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}