feat: add a simple table to show recently indexed URLs

backend/app/db.py

@@ -23,7 +23,7 @@ def increment_usage_counter(self) -> int:
     def search_pages(
         self, emb: list[float], user_id: str, threshold: float = 0.5, top_n: int = 10
     ):
-        data = self._client.rpc(
+        result = self._client.rpc(
             "search_pages",
             {
                 "query_embedding": emb,
@@ -32,12 +32,12 @@ def search_pages(
                 "top_n": top_n,
             },
         ).execute()
-        return data.data
+        return result.data
 
     def search_chunks(
         self, emb: list[float], user_id: str, threshold: float = 0.5, top_n: int = 10
     ):
-        data = self._client.rpc(
+        result = self._client.rpc(
             "search_chunks",
             {
                 "query_embedding": emb,
@@ -46,22 +46,22 @@ def search_chunks(
                 "top_n": top_n,
             },
         ).execute()
-        return data.data
+        return result.data
 
     def get_text_node(self, id: str):
-        data = (
+        result = (
             self._client.table("text_nodes")
             .select("id, title, text, summary, url, embedding")
             .eq("id", id)
             .execute()
         )
-        return data.data[0]
+        return result.data[0]
 
     def get_similar_text_nodes(self, id: str, top_n: int = 10):
-        data = self._client.rpc(
+        result = self._client.rpc(
             "get_similar_text_nodes", {"id": id, "top_n": top_n}
         ).execute()
-        return data.data
+        return result.data
 
     def create_urls(self, urls: list[URL], user_id: str):
         urls_to_persist = []
@@ -101,3 +101,14 @@ def create_text_nodes(self, nodes: list[TextNode], user_id: str):
         self._client.table("text_node_chunks").insert(
             text_node_chunks_to_persist
         ).execute()
+
+    def get_urls_feed(self, user_id: str):
+        result = (
+            self._client.table("urls_feed")
+            .select("id, created_at, url, status")
+            .eq("user_id", user_id)
+            .order("created_at", desc=True)
+            .limit(10)
+            .execute()
+        )
+        return result.data

backend/app/main.py

@@ -96,7 +96,7 @@ async def get_node_route(id: str, user=Depends(get_current_user)):
 
     node = db.get_text_node(id)
     related_nodes = db.search_pages(
-        node["embedding"], user_id=user_id, threshold=0.6, top_n=5
+        node["embedding"], user_id=user_id, threshold=0.4, top_n=5
     )
     related_nodes = [n for n in related_nodes if n["id"] != id]
     node["related"] = related_nodes
@@ -105,6 +105,13 @@ async def get_node_route(id: str, user=Depends(get_current_user)):
     return node
 
 
+@app.get("/api/index-feed")
+async def get_index_feed_route(user=Depends(get_current_user)):
+    user_id = user.id
+    urls = db.get_urls_feed(user_id)
+    return urls
+
+
 @app.post("/api/index")
 async def post_index_route(payload: PageCreate, user=Depends(get_current_user)):
     try:

frontend/package.json

@@ -18,6 +18,7 @@
   },
   "dependencies": {
     "@headlessui/vue": "^1.7.21",
+    "date-fns": "^3.6.0",
     "markdown-it": "^14.1.0"
   }
 }

frontend/pages/index.vue

@@ -1,16 +1,27 @@
 <script setup lang="ts">
+import { formatDistance } from 'date-fns';
+
+interface IndexFeedResult {
+  id: string;
+  created_at: string;
+  status: string;
+  url: string;
+}
+
 definePageMeta({ path: '/index' });
 
 const input = ref('');
 const urlsCount = ref(0);
 const urls = ref();
 const indexingStatusMessage = ref('');
+const indexFeedResults = ref<IndexFeedResult[]>([]);
 
 const config = useRuntimeConfig();
 const apiBase = config.public.apiBase;
 
 const indexWebPages = async () => {
   indexingStatusMessage.value = 'Indexing...';
+  const interval = startPollingIndexFeed();
   const token = useSupabaseSession().value?.access_token;
   // TODO: handle re-auth
   if (!token) return;
@@ -31,9 +42,50 @@ const indexWebPages = async () => {
     urlsCount.value = 0;
     urls.value = [];
     indexingStatusMessage.value = '';
+    clearInterval(interval);
   }, 3000);
 };
 
+const fetchIndexedPages = async () => {
+  const token = useSupabaseSession().value?.access_token;
+  // TODO: handle re-auth
+  if (!token) return;
+  const { data } = await useFetch<[IndexFeedResult]>(
+    `${apiBase}/api/index-feed`,
+    {
+      method: 'GET',
+      headers: {
+        Authorization: `Bearer ${token}`,
+      },
+    }
+  );
+  if (data.value) {
+    indexFeedResults.value = data.value;
+  } else {
+    indexFeedResults.value = [];
+  }
+};
+
+const startPollingIndexFeed = () => {
+  const interval = setInterval(async () => {
+    await fetchIndexedPages();
+  }, 10000);
+
+  onUnmounted(() => {
+    clearInterval(interval);
+  });
+
+  return interval;
+};
+
+const formatTimeToHumanFriendly = (time: string) => {
+  return formatDistance(new Date(time), new Date(), { addSuffix: true });
+};
+
+onMounted(async () => {
+  await fetchIndexedPages();
+});
+
 watch(input, (newValue) => {
   const inputUrls = newValue.split(/[\n,]+/).filter(Boolean);
   urlsCount.value = inputUrls.length;
@@ -42,6 +94,7 @@ watch(input, (newValue) => {
 </script>
 
 <template>
+  <!-- Indexing Input Textarea -->
   <div class="relative mt-2">
     <textarea
       placeholder="Enter the URL(s) you want to index, separated by commas or newlines."
@@ -76,4 +129,35 @@ watch(input, (newValue) => {
       Index
     </button>
   </div>
+
+  <!-- URL Index Feed Table -->
+  <table
+    class="rounded-md mt-2 border-collapse table-auto w-full"
+    v-if="indexFeedResults.length"
+  >
+    <thead class="bg-gray-200">
+      <tr>
+        <th class="rounded-tl-md py-2 px-4 text-left">URL</th>
+        <th class="py-2 px-4 text-left">Submitted</th>
+        <th class="rounded-tr-md py-2 px-4 text-left">Status</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr
+        v-for="(item, index) in indexFeedResults"
+        :key="index"
+        class="border-b border-slate-200 hover:bg-gray-100 cursor-pointer"
+      >
+        <td class="py-2 px-4 text-xs text-slate-600">
+          <a :href="item.url" target="_blank" class="text-xs" @click.stop>
+            {{ item.url }}
+          </a>
+        </td>
+        <td class="py-2 px-4 text-xs text-slate-600">
+          {{ formatTimeToHumanFriendly(item.created_at) }}
+        </td>
+        <td class="py-2 px-4 text-xs text-slate-600">{{ item.status }}</td>
+      </tr>
+    </tbody>
+  </table>
 </template>

frontend/pages/search.vue

@@ -1,12 +1,12 @@
 <script setup lang="ts">
-interface Result {
+interface SearchResult {
   id: string;
   title: string;
   url: string;
   score: number;
 }
 
-const results = ref<Result[]>([]);
+const results = ref<SearchResult[]>([]);
 const isLoading = ref(false);
 const isResultsEmpty = ref(false);
 const lastSearchQuery = ref('');
@@ -25,7 +25,7 @@ const search = async (query: string) => {
   const token = useSupabaseSession().value?.access_token;
   // TODO: handle re-auth
   if (!token) return;
-  const { data } = await useFetch<Result[]>(`${apiBase}/api/search`, {
+  const { data } = await useFetch<SearchResult[]>(`${apiBase}/api/search`, {
     method: 'GET',
     query: { q: query },
     headers: { Authorization: `Bearer ${token}` },