Getting playwright started and refactoring article fetch plus tests

playwright.config.ts

@@ -6,7 +6,10 @@ const config: PlaywrightTestConfig = {
 		port: 4173
 	},
 	testDir: 'tests',
-	testMatch: /(.+\.)?(test|spec)\.[jt]s/
+	testMatch: /(.+\.)?(test|spec)\.[jt]s/,
+	use: {
+		baseURL: 'http://localhost:4173'
+	}
 };
 
 export default config;

src/index.test.ts

@@ -1,7 +0,0 @@
-import { describe, it, expect } from 'vitest';
-
-describe('sum test', () => {
-	it('adds 1 + 2 to equal 3', () => {
-		expect(1 + 2).toBe(3);
-	});
-});

src/lib/services/articlesApi.test.ts

@@ -0,0 +1,131 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+// Mock private env used by the service
+vi.mock('$env/static/private', () => ({
+  PAGE_SIZE: '10',
+  USE_REDIS_CACHE: 'true',
+  WALLABAG_CLIENT_ID: 'cid',
+  WALLABAG_CLIENT_SECRET: 'csecret',
+  WALLABAG_PASSWORD: 'pw',
+  WALLABAG_URL: 'https://example.com',
+  WALLABAG_USERNAME: 'user',
+}));
+
+// Mock redis client
+const redisGet = vi.fn();
+const redisSet = vi.fn();
+const redisTtl = vi.fn();
+vi.mock('$lib/server/redis', () => ({
+  redis: {
+    get: (key: string) => redisGet(key),
+    set: (key: string, value: string, mode: 'EX', seconds: number) => redisSet(key, value, mode, seconds),
+    ttl: (key: string) => redisTtl(key),
+  },
+}));
+
+import { fetchArticlesApi } from './articlesApi';
+
+type MockResponse<T> = {
+  ok: boolean;
+  headers?: { get: (k: string) => string | null };
+  json: () => Promise<T>;
+};
+
+const makeWallabagResponse = () => ({
+  _embedded: {
+    items: [
+      {
+        title: 'Article 1',
+        url: 'https://example.com/a1',
+        domain_name: 'www.example.com',
+        hashed_url: 'hash1',
+        reading_time: 5,
+        preview_picture: 'https://example.com/img.jpg',
+        created_at: new Date().toISOString(),
+        updated_at: new Date().toISOString(),
+        archived_at: null,
+        tags: [{ id: 1, label: 'Programming', slug: 'programming' }],
+      },
+    ],
+  },
+  page: 1,
+  pages: 1,
+  total: 1,
+  limit: 10,
+});
+
+const makeCachedResponse = () => ({
+  articles: [],
+  currentPage: 1,
+  totalPages: 1,
+  limit: 10,
+  totalArticles: 0,
+  cacheControl: 'max-age=60',
+});
+
+const originalFetch: typeof globalThis.fetch = globalThis.fetch;
+
+beforeEach(() => {
+  vi.useFakeTimers();
+  redisGet.mockReset();
+  redisSet.mockReset();
+  redisTtl.mockReset();
+});
+
+afterEach(() => {
+  vi.useRealTimers();
+  globalThis.fetch = originalFetch;
+});
+
+describe('fetchArticlesApi', () => {
+  it('returns cached response with cacheControl on cache hit', async () => {
+    const cached = makeCachedResponse();
+    redisGet.mockResolvedValueOnce(JSON.stringify(cached));
+    redisTtl.mockResolvedValueOnce(60);
+
+    // fetch should not be called on cache hit
+    globalThis.fetch = vi.fn() as unknown as typeof globalThis.fetch;
+
+    const result = await fetchArticlesApi('get', 'fetchArticles', { page: '1', limit: '10' });
+
+    expect(result).toBeTruthy();
+    expect(result.cacheControl).toBe('max-age=60');
+    expect(redisGet).toHaveBeenCalledTimes(1);
+    expect(global.fetch).not.toHaveBeenCalled();
+  });
+
+  it('fetches from API and stores in cache on cache miss', async () => {
+    // Cache miss
+    redisGet.mockResolvedValueOnce(null);
+
+    // Mock fetch for auth and entries
+    const authResponse: MockResponse<{ access_token: string }> = {
+      ok: true,
+      json: async () => ({ access_token: 'token' }),
+    };
+
+    const entriesJson = makeWallabagResponse();
+    const pageResponse: MockResponse<ReturnType<typeof makeWallabagResponse>> = {
+      ok: true,
+      headers: { get: (k: string) => (k.toLowerCase() === 'cache-control' ? 'max-age=120' : null) },
+      json: async () => entriesJson,
+    };
+
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(authResponse) // oauth token
+      .mockResolvedValueOnce(pageResponse); // entries
+
+    globalThis.fetch = fetchMock as unknown as typeof globalThis.fetch;
+
+    const result = await fetchArticlesApi('get', 'fetchArticles', { page: '1', limit: '10' });
+
+    expect(result).toBeTruthy();
+    expect(result.cacheControl).toBe('max-age=120');
+    expect(result.articles.length).toBeGreaterThan(0);
+    expect(result.articles[0].domain_name).toBe('example.com');
+
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    expect(redisSet).toHaveBeenCalled();
+  });
+});

src/lib/services/articlesApi.ts

@@ -0,0 +1,196 @@
+import intersect from 'just-intersect';
+import {
+  PAGE_SIZE,
+  USE_REDIS_CACHE,
+  WALLABAG_CLIENT_ID,
+  WALLABAG_CLIENT_SECRET,
+  WALLABAG_PASSWORD,
+  WALLABAG_URL,
+  WALLABAG_USERNAME,
+} from '$env/static/private';
+import { redis } from '$lib/server/redis';
+import type { Article, ArticlePageLoad, WallabagArticle } from '$lib/types/article';
+import { ArticleTag } from '$lib/types/articleTag';
+import type { PageQuery } from '../types/pageQuery';
+
+const base: string = WALLABAG_URL;
+
+// Retry helper with exponential backoff
+async function retryWithBackoff<T>(fn: () => Promise<T>, maxRetries = 3, baseDelay = 500): Promise<T> {
+  let lastError: Error | undefined;
+
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error as Error;
+
+      if (attempt === maxRetries) {
+        throw lastError;
+      }
+
+      // Exponential backoff: 500ms, 1s, 2s
+      const delay = baseDelay * 2 ** attempt;
+      console.log(`Attempt ${attempt + 1} failed, retrying in ${delay}ms...`);
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
+
+  throw lastError;
+}
+
+export async function fetchArticlesApi(_method: string, _resource: string, queryParams: Record<string, string>) {
+  try {
+    let perPage = Number(queryParams?.limit);
+    if (perPage === undefined || perPage > 30 || perPage < 1) {
+      perPage = Number(PAGE_SIZE);
+    } else {
+      perPage = Number(queryParams?.limit);
+    }
+
+    const pageQuery: PageQuery = {
+      sort: 'updated',
+      perPage,
+      since: 0,
+      page: Number(queryParams?.page) || 1,
+      tags: 'programming',
+      content: 'metadata',
+    };
+    const entriesQueryParams = new URLSearchParams({
+      sort: pageQuery.sort,
+      perPage: `${pageQuery.perPage}`,
+      since: `${pageQuery.since}`,
+      page: `${pageQuery.page}`,
+      tags: pageQuery.tags,
+      content: pageQuery.content,
+    });
+
+    if (USE_REDIS_CACHE === 'true') {
+      console.log('Using redis cache');
+      const cacheKey = entriesQueryParams.toString();
+      console.log(`Cache key: ${cacheKey}`);
+      const cached = await redis.get(cacheKey);
+
+      if (cached) {
+        console.log('Cache hit!');
+        const response = JSON.parse(cached);
+        const ttl = await redis.ttl(cacheKey);
+
+        console.log(`Returning cached response for page ${pageQuery.page} with ttl of ${ttl} seconds`);
+        console.log(`Response: ${JSON.stringify(response)}`);
+        return { ...response, cacheControl: `max-age=${ttl}` };
+      }
+      console.log(`Cache miss for page ${pageQuery.page}, fetching from API`);
+    }
+
+    const authBody = {
+      grant_type: 'password',
+      client_id: WALLABAG_CLIENT_ID,
+      client_secret: WALLABAG_CLIENT_SECRET,
+      username: WALLABAG_USERNAME,
+      password: WALLABAG_PASSWORD,
+    };
+
+    console.log(`Auth body: ${JSON.stringify(authBody)}`);
+
+    const auth = await retryWithBackoff(async () => {
+      const authResponse = await fetch(`${base}/oauth/v2/token`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+        body: new URLSearchParams(authBody),
+        signal: AbortSignal.timeout(10000), // 10 second timeout
+      });
+
+      if (!authResponse.ok) {
+        throw new Error(`Auth failed: ${authResponse.status} ${authResponse.statusText}`);
+      }
+
+      return await authResponse.json();
+    });
+
+    console.log(`Got auth response: ${JSON.stringify(auth)}`);
+
+    const { wallabagResponse, cacheControl } = await retryWithBackoff(async () => {
+      const pageResponse = await fetch(`${WALLABAG_URL}/api/entries.json?${entriesQueryParams}`, {
+        method: 'GET',
+        headers: {
+          Authorization: `Bearer ${auth.access_token}`,
+        },
+        signal: AbortSignal.timeout(15000), // 15 second timeout
+      });
+
+      console.log('pageResponse', pageResponse);
+
+      if (!pageResponse.ok) {
+        console.log('pageResponse not ok', pageResponse);
+        throw new Error(`API request failed: ${pageResponse.status} ${pageResponse.statusText}`);
+      }
+
+      const cacheControl = pageResponse.headers.get('cache-control') || 'no-cache';
+      const wallabagResponse = await pageResponse.json();
+
+      return { wallabagResponse, cacheControl };
+    });
+    console.log('wallabagResponse', JSON.stringify(wallabagResponse));
+    const { _embedded: favoriteArticles, page, pages, total, limit } = wallabagResponse;
+    const articles: Article[] = [];
+
+    console.log('favoriteArticles', JSON.stringify(favoriteArticles));
+    console.log('pages', pages);
+    console.log('page', page);
+    console.log('total', total);
+    console.log('limit', limit);
+
+    for (const article of favoriteArticles.items as WallabagArticle[]) {
+      const rawTags = article?.tags?.map((tag) => tag.slug);
+      if (intersect(rawTags, Object.values(ArticleTag))?.length > 0) {
+        const tags = rawTags.map((rawTag) => rawTag as unknown as ArticleTag);
+        articles.push({
+          tags,
+          title: article.title,
+          url: new URL(article.url),
+          domain_name: article.domain_name?.replace('www.', '') ?? '',
+          hashed_url: article.hashed_url,
+          reading_time: article.reading_time,
+          preview_picture: article.preview_picture,
+          created_at: new Date(article.created_at),
+          updated_at: new Date(article.updated_at),
+          archived_at: article.archived_at ? new Date(article.archived_at) : null,
+        });
+      }
+    }
+
+    const responseData: ArticlePageLoad = {
+      articles,
+      currentPage: page,
+      totalPages: pages,
+      limit,
+      totalArticles: total,
+      cacheControl,
+    };
+
+    console.log('Response data from API: ', JSON.stringify(responseData));
+
+    if (USE_REDIS_CACHE === 'true' && responseData?.articles?.length > 0) {
+      const cacheKey = entriesQueryParams.toString();
+      console.log(`Storing in cache with key: ${cacheKey} for page ${page}`);
+      redis.set(cacheKey, JSON.stringify(responseData), 'EX', 43200);
+    }
+
+    return responseData;
+  } catch (error) {
+    console.error(`Error fetching articles for page ${queryParams?.page}:`, error);
+
+    // Return empty response on error to prevent app crash
+    const fallbackResponse: ArticlePageLoad = {
+      articles: [],
+      currentPage: Number(queryParams?.page) || 1,
+      totalPages: 0,
+      limit: Number(queryParams?.limit) || Number(PAGE_SIZE),
+      totalArticles: 0,
+      cacheControl: 'no-cache',
+    };
+
+    return fallbackResponse;
+  }
+}

src/lib/util/fetchBandcampAlbums.test.ts

@@ -1,17 +1,17 @@
-import { describe, it, expect } from 'vitest';
+import { describe, expect, it } from 'vitest';
 import { fetchBandcampAlbums } from './fetchBandcampAlbums';
 
 describe('test fetchBandcampAlbums', () => {
-	it('fetches bandcamp albums', async () => {
+  it('fetches bandcamp albums', async () => {
-		const albums = await fetchBandcampAlbums();
+    const albums = await fetchBandcampAlbums();
-		expect(albums).not.toBeNull();
+    expect(albums).not.toBeNull();
-		expect(albums).toBeTruthy();
+    expect(albums).toBeTruthy();
-		expect(albums?.length).toBeGreaterThan(0);
+    expect(albums?.length).toBeGreaterThan(0);
-		for (const album of albums) {
+    for (const album of albums) {
-			expect(album?.artist).toHaveLength;
+      expect(album?.artist).toHaveLength;
-			expect(album?.artwork).toHaveLength;
+      expect(album?.artwork).toHaveLength;
-			expect(album?.title).toHaveLength;
+      expect(album?.title).toHaveLength;
-			expect(album?.url).toHaveLength;
+      expect(album?.url).toHaveLength;
-		}
+    }
-	});
+  });
 });

src/lib/util/fetchBandcampAlbums.ts

@@ -1,60 +1,69 @@
-import { BANDCAMP_USERNAME, USE_REDIS_CACHE } from '$env/static/private';
-import scrapeIt from 'scrape-it';
 import type { ScrapeResult } from 'scrape-it';
+import scrapeIt from 'scrape-it';
+import { BANDCAMP_USERNAME, USE_REDIS_CACHE } from '$env/static/private';
 import { redis } from '$lib/server/redis';
 import type { Album, BandCampResults } from '../types/album';
 
-export async function fetchBandcampAlbums() {
+export async function fetchBandcampAlbums(): Promise<Album[] & { cacheControl?: string }> {
-	try {
+  try {
-		if (USE_REDIS_CACHE === 'true') {
+    if (USE_REDIS_CACHE === 'true') {
-			const cached: string | null = await redis.get('bandcampAlbums');
+      const cached: string | null = await redis.get('bandcampAlbums');
 
-			if (cached) {
+      if (cached) {
-				const response: Album[] = JSON.parse(cached);
+        const response: Album[] = JSON.parse(cached);
-				console.log('Cache hit!');
+        console.log('Cache hit!');
-				const ttl = await redis.ttl('bandcampAlbums');
+        const ttl = await redis.ttl('bandcampAlbums');
 
-				return { ...response, cacheControl: `max-age=${ttl}` };
+        // Preserve array shape; attach cacheControl as a non-enumerable property.
-			}
+        if (typeof ttl === 'number' && ttl > 0) {
-		}
+          Object.defineProperty(response, 'cacheControl', {
+            value: `max-age=${ttl}`,
+            enumerable: false,
+          });
+        }
+        return response as Album[] & { cacheControl?: string };
+      }
+    }
 
-		const { data }: ScrapeResult<BandCampResults> = await scrapeIt(
+    const { data }: ScrapeResult<BandCampResults> = await scrapeIt(`https://bandcamp.com/${BANDCAMP_USERNAME}`, {
-			`https://bandcamp.com/${BANDCAMP_USERNAME}`,
+      collectionItems: {
-			{
+        listItem: '.collection-item-container',
-				collectionItems: {
+        data: {
-					listItem: '.collection-item-container',
+          url: {
-					data: {
+            selector: '.collection-title-details > a.item-link',
-						url: {
+            attr: 'href',
-							selector: '.collection-title-details > a.item-link',
+          },
-							attr: 'href'
+          artwork: {
-						},
+            selector: 'div.collection-item-art-container a img',
-						artwork: {
+            attr: 'src',
-							selector: 'div.collection-item-art-container a img',
+          },
-							attr: 'src'
+          title: {
-						},
+            selector: 'span.item-link-alt > div.collection-item-title',
-						title: {
+          },
-							selector: 'span.item-link-alt > div.collection-item-title'
+          artist: {
-						},
+            selector: 'span.item-link-alt > div.collection-item-artist',
-						artist: {
+          },
-							selector: 'span.item-link-alt > div.collection-item-artist'
+        },
-						}
+      },
-					}
+    });
-				}
-			}
-		);
 
-		const albums: Album[] = data?.collectionItems || [];
+    const albums: Album[] = data?.collectionItems || [];
 
-		if (albums && albums?.length > 0) {
+    if (albums && albums?.length > 0) {
-			if (USE_REDIS_CACHE === 'true') {
+      if (USE_REDIS_CACHE === 'true') {
-				redis.set('bandcampAlbums', JSON.stringify(albums), 'EX', 43200);
+        // Store in Redis for 12 hours.
-			}
+        redis.set('bandcampAlbums', JSON.stringify(albums), 'EX', 43200);
-			return albums;
+        // Reflect the cache TTL on the returned array as a hint to clients.
-		} else {
+        Object.defineProperty(albums, 'cacheControl', {
-			return [];
+          value: 'max-age=43200',
-		}
+          enumerable: false,
-	} catch (error) {
+        });
-		console.error(error);
+      }
-		return [];
+      return albums as Album[] & { cacheControl?: string };
-	}
+    }
+    return [] as Album[] & { cacheControl?: string };
+  } catch (error) {
+    console.error(error);
+    return [] as Album[] & { cacheControl?: string };
+  }
 }

src/routes/api/articles/+server.ts

@@ -1,6 +1,6 @@
 import { json, error } from '@sveltejs/kit';
 import { PAGE_SIZE } from '$env/static/private';
-import { fetchArticlesApi } from '$lib/api';
+import { fetchArticlesApi } from '$lib/services/articlesApi';
 import type { ArticlePageLoad } from '@/lib/types/article.js';
 
 export async function GET({ setHeaders, url }) {

test-results/.last-run.json

@@ -0,0 +1,4 @@
+{
+  "status": "passed",
+  "failedTests": []
+}