diff --git a/playwright.config.ts b/playwright.config.ts index 3cf8326..ba1e9fd 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -6,7 +6,10 @@ const config: PlaywrightTestConfig = { port: 4173 }, testDir: 'tests', - testMatch: /(.+\.)?(test|spec)\.[jt]s/ + testMatch: /(.+\.)?(test|spec)\.[jt]s/, + use: { + baseURL: 'http://localhost:4173' + } }; export default config; diff --git a/src/index.test.ts b/src/index.test.ts deleted file mode 100644 index e07cbbd..0000000 --- a/src/index.test.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { describe, it, expect } from 'vitest'; - -describe('sum test', () => { - it('adds 1 + 2 to equal 3', () => { - expect(1 + 2).toBe(3); - }); -}); diff --git a/src/lib/services/articlesApi.test.ts b/src/lib/services/articlesApi.test.ts new file mode 100644 index 0000000..f4ca659 --- /dev/null +++ b/src/lib/services/articlesApi.test.ts @@ -0,0 +1,131 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +// Mock private env used by the service +vi.mock('$env/static/private', () => ({ + PAGE_SIZE: '10', + USE_REDIS_CACHE: 'true', + WALLABAG_CLIENT_ID: 'cid', + WALLABAG_CLIENT_SECRET: 'csecret', + WALLABAG_PASSWORD: 'pw', + WALLABAG_URL: 'https://example.com', + WALLABAG_USERNAME: 'user', +})); + +// Mock redis client +const redisGet = vi.fn(); +const redisSet = vi.fn(); +const redisTtl = vi.fn(); +vi.mock('$lib/server/redis', () => ({ + redis: { + get: (key: string) => redisGet(key), + set: (key: string, value: string, mode: 'EX', seconds: number) => redisSet(key, value, mode, seconds), + ttl: (key: string) => redisTtl(key), + }, +})); + +import { fetchArticlesApi } from './articlesApi'; + +type MockResponse = { + ok: boolean; + headers?: { get: (k: string) => string | null }; + json: () => Promise; +}; + +const makeWallabagResponse = () => ({ + _embedded: { + items: [ + { + title: 'Article 1', + url: 'https://example.com/a1', + domain_name: 'www.example.com', + hashed_url: 'hash1', + reading_time: 5, + preview_picture: 'https://example.com/img.jpg', + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + archived_at: null, + tags: [{ id: 1, label: 'Programming', slug: 'programming' }], + }, + ], + }, + page: 1, + pages: 1, + total: 1, + limit: 10, +}); + +const makeCachedResponse = () => ({ + articles: [], + currentPage: 1, + totalPages: 1, + limit: 10, + totalArticles: 0, + cacheControl: 'max-age=60', +}); + +const originalFetch: typeof globalThis.fetch = globalThis.fetch; + +beforeEach(() => { + vi.useFakeTimers(); + redisGet.mockReset(); + redisSet.mockReset(); + redisTtl.mockReset(); +}); + +afterEach(() => { + vi.useRealTimers(); + globalThis.fetch = originalFetch; +}); + +describe('fetchArticlesApi', () => { + it('returns cached response with cacheControl on cache hit', async () => { + const cached = makeCachedResponse(); + redisGet.mockResolvedValueOnce(JSON.stringify(cached)); + redisTtl.mockResolvedValueOnce(60); + + // fetch should not be called on cache hit + globalThis.fetch = vi.fn() as unknown as typeof globalThis.fetch; + + const result = await fetchArticlesApi('get', 'fetchArticles', { page: '1', limit: '10' }); + + expect(result).toBeTruthy(); + expect(result.cacheControl).toBe('max-age=60'); + expect(redisGet).toHaveBeenCalledTimes(1); + expect(global.fetch).not.toHaveBeenCalled(); + }); + + it('fetches from API and stores in cache on cache miss', async () => { + // Cache miss + redisGet.mockResolvedValueOnce(null); + + // Mock fetch for auth and entries + const authResponse: MockResponse<{ access_token: string }> = { + ok: true, + json: async () => ({ access_token: 'token' }), + }; + + const entriesJson = makeWallabagResponse(); + const pageResponse: MockResponse> = { + ok: true, + headers: { get: (k: string) => (k.toLowerCase() === 'cache-control' ? 'max-age=120' : null) }, + json: async () => entriesJson, + }; + + const fetchMock = vi + .fn() + .mockResolvedValueOnce(authResponse) // oauth token + .mockResolvedValueOnce(pageResponse); // entries + + globalThis.fetch = fetchMock as unknown as typeof globalThis.fetch; + + const result = await fetchArticlesApi('get', 'fetchArticles', { page: '1', limit: '10' }); + + expect(result).toBeTruthy(); + expect(result.cacheControl).toBe('max-age=120'); + expect(result.articles.length).toBeGreaterThan(0); + expect(result.articles[0].domain_name).toBe('example.com'); + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(redisSet).toHaveBeenCalled(); + }); +}); diff --git a/src/lib/services/articlesApi.ts b/src/lib/services/articlesApi.ts new file mode 100644 index 0000000..6d92fbf --- /dev/null +++ b/src/lib/services/articlesApi.ts @@ -0,0 +1,196 @@ +import intersect from 'just-intersect'; +import { + PAGE_SIZE, + USE_REDIS_CACHE, + WALLABAG_CLIENT_ID, + WALLABAG_CLIENT_SECRET, + WALLABAG_PASSWORD, + WALLABAG_URL, + WALLABAG_USERNAME, +} from '$env/static/private'; +import { redis } from '$lib/server/redis'; +import type { Article, ArticlePageLoad, WallabagArticle } from '$lib/types/article'; +import { ArticleTag } from '$lib/types/articleTag'; +import type { PageQuery } from '../types/pageQuery'; + +const base: string = WALLABAG_URL; + +// Retry helper with exponential backoff +async function retryWithBackoff(fn: () => Promise, maxRetries = 3, baseDelay = 500): Promise { + let lastError: Error | undefined; + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + return await fn(); + } catch (error) { + lastError = error as Error; + + if (attempt === maxRetries) { + throw lastError; + } + + // Exponential backoff: 500ms, 1s, 2s + const delay = baseDelay * 2 ** attempt; + console.log(`Attempt ${attempt + 1} failed, retrying in ${delay}ms...`); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + + throw lastError; +} + +export async function fetchArticlesApi(_method: string, _resource: string, queryParams: Record) { + try { + let perPage = Number(queryParams?.limit); + if (perPage === undefined || perPage > 30 || perPage < 1) { + perPage = Number(PAGE_SIZE); + } else { + perPage = Number(queryParams?.limit); + } + + const pageQuery: PageQuery = { + sort: 'updated', + perPage, + since: 0, + page: Number(queryParams?.page) || 1, + tags: 'programming', + content: 'metadata', + }; + const entriesQueryParams = new URLSearchParams({ + sort: pageQuery.sort, + perPage: `${pageQuery.perPage}`, + since: `${pageQuery.since}`, + page: `${pageQuery.page}`, + tags: pageQuery.tags, + content: pageQuery.content, + }); + + if (USE_REDIS_CACHE === 'true') { + console.log('Using redis cache'); + const cacheKey = entriesQueryParams.toString(); + console.log(`Cache key: ${cacheKey}`); + const cached = await redis.get(cacheKey); + + if (cached) { + console.log('Cache hit!'); + const response = JSON.parse(cached); + const ttl = await redis.ttl(cacheKey); + + console.log(`Returning cached response for page ${pageQuery.page} with ttl of ${ttl} seconds`); + console.log(`Response: ${JSON.stringify(response)}`); + return { ...response, cacheControl: `max-age=${ttl}` }; + } + console.log(`Cache miss for page ${pageQuery.page}, fetching from API`); + } + + const authBody = { + grant_type: 'password', + client_id: WALLABAG_CLIENT_ID, + client_secret: WALLABAG_CLIENT_SECRET, + username: WALLABAG_USERNAME, + password: WALLABAG_PASSWORD, + }; + + console.log(`Auth body: ${JSON.stringify(authBody)}`); + + const auth = await retryWithBackoff(async () => { + const authResponse = await fetch(`${base}/oauth/v2/token`, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams(authBody), + signal: AbortSignal.timeout(10000), // 10 second timeout + }); + + if (!authResponse.ok) { + throw new Error(`Auth failed: ${authResponse.status} ${authResponse.statusText}`); + } + + return await authResponse.json(); + }); + + console.log(`Got auth response: ${JSON.stringify(auth)}`); + + const { wallabagResponse, cacheControl } = await retryWithBackoff(async () => { + const pageResponse = await fetch(`${WALLABAG_URL}/api/entries.json?${entriesQueryParams}`, { + method: 'GET', + headers: { + Authorization: `Bearer ${auth.access_token}`, + }, + signal: AbortSignal.timeout(15000), // 15 second timeout + }); + + console.log('pageResponse', pageResponse); + + if (!pageResponse.ok) { + console.log('pageResponse not ok', pageResponse); + throw new Error(`API request failed: ${pageResponse.status} ${pageResponse.statusText}`); + } + + const cacheControl = pageResponse.headers.get('cache-control') || 'no-cache'; + const wallabagResponse = await pageResponse.json(); + + return { wallabagResponse, cacheControl }; + }); + console.log('wallabagResponse', JSON.stringify(wallabagResponse)); + const { _embedded: favoriteArticles, page, pages, total, limit } = wallabagResponse; + const articles: Article[] = []; + + console.log('favoriteArticles', JSON.stringify(favoriteArticles)); + console.log('pages', pages); + console.log('page', page); + console.log('total', total); + console.log('limit', limit); + + for (const article of favoriteArticles.items as WallabagArticle[]) { + const rawTags = article?.tags?.map((tag) => tag.slug); + if (intersect(rawTags, Object.values(ArticleTag))?.length > 0) { + const tags = rawTags.map((rawTag) => rawTag as unknown as ArticleTag); + articles.push({ + tags, + title: article.title, + url: new URL(article.url), + domain_name: article.domain_name?.replace('www.', '') ?? '', + hashed_url: article.hashed_url, + reading_time: article.reading_time, + preview_picture: article.preview_picture, + created_at: new Date(article.created_at), + updated_at: new Date(article.updated_at), + archived_at: article.archived_at ? new Date(article.archived_at) : null, + }); + } + } + + const responseData: ArticlePageLoad = { + articles, + currentPage: page, + totalPages: pages, + limit, + totalArticles: total, + cacheControl, + }; + + console.log('Response data from API: ', JSON.stringify(responseData)); + + if (USE_REDIS_CACHE === 'true' && responseData?.articles?.length > 0) { + const cacheKey = entriesQueryParams.toString(); + console.log(`Storing in cache with key: ${cacheKey} for page ${page}`); + redis.set(cacheKey, JSON.stringify(responseData), 'EX', 43200); + } + + return responseData; + } catch (error) { + console.error(`Error fetching articles for page ${queryParams?.page}:`, error); + + // Return empty response on error to prevent app crash + const fallbackResponse: ArticlePageLoad = { + articles: [], + currentPage: Number(queryParams?.page) || 1, + totalPages: 0, + limit: Number(queryParams?.limit) || Number(PAGE_SIZE), + totalArticles: 0, + cacheControl: 'no-cache', + }; + + return fallbackResponse; + } +} diff --git a/src/lib/util/fetchBandcampAlbums.test.ts b/src/lib/util/fetchBandcampAlbums.test.ts index 09ddd5a..1912af3 100644 --- a/src/lib/util/fetchBandcampAlbums.test.ts +++ b/src/lib/util/fetchBandcampAlbums.test.ts @@ -1,17 +1,17 @@ -import { describe, it, expect } from 'vitest'; +import { describe, expect, it } from 'vitest'; import { fetchBandcampAlbums } from './fetchBandcampAlbums'; describe('test fetchBandcampAlbums', () => { - it('fetches bandcamp albums', async () => { - const albums = await fetchBandcampAlbums(); - expect(albums).not.toBeNull(); - expect(albums).toBeTruthy(); - expect(albums?.length).toBeGreaterThan(0); - for (const album of albums) { - expect(album?.artist).toHaveLength; - expect(album?.artwork).toHaveLength; - expect(album?.title).toHaveLength; - expect(album?.url).toHaveLength; - } - }); + it('fetches bandcamp albums', async () => { + const albums = await fetchBandcampAlbums(); + expect(albums).not.toBeNull(); + expect(albums).toBeTruthy(); + expect(albums?.length).toBeGreaterThan(0); + for (const album of albums) { + expect(album?.artist).toHaveLength; + expect(album?.artwork).toHaveLength; + expect(album?.title).toHaveLength; + expect(album?.url).toHaveLength; + } + }); }); diff --git a/src/lib/util/fetchBandcampAlbums.ts b/src/lib/util/fetchBandcampAlbums.ts index 29499a5..5607ac0 100644 --- a/src/lib/util/fetchBandcampAlbums.ts +++ b/src/lib/util/fetchBandcampAlbums.ts @@ -1,60 +1,69 @@ -import { BANDCAMP_USERNAME, USE_REDIS_CACHE } from '$env/static/private'; -import scrapeIt from 'scrape-it'; import type { ScrapeResult } from 'scrape-it'; +import scrapeIt from 'scrape-it'; +import { BANDCAMP_USERNAME, USE_REDIS_CACHE } from '$env/static/private'; import { redis } from '$lib/server/redis'; import type { Album, BandCampResults } from '../types/album'; -export async function fetchBandcampAlbums() { - try { - if (USE_REDIS_CACHE === 'true') { - const cached: string | null = await redis.get('bandcampAlbums'); +export async function fetchBandcampAlbums(): Promise { + try { + if (USE_REDIS_CACHE === 'true') { + const cached: string | null = await redis.get('bandcampAlbums'); - if (cached) { - const response: Album[] = JSON.parse(cached); - console.log('Cache hit!'); - const ttl = await redis.ttl('bandcampAlbums'); + if (cached) { + const response: Album[] = JSON.parse(cached); + console.log('Cache hit!'); + const ttl = await redis.ttl('bandcampAlbums'); - return { ...response, cacheControl: `max-age=${ttl}` }; - } - } + // Preserve array shape; attach cacheControl as a non-enumerable property. + if (typeof ttl === 'number' && ttl > 0) { + Object.defineProperty(response, 'cacheControl', { + value: `max-age=${ttl}`, + enumerable: false, + }); + } + return response as Album[] & { cacheControl?: string }; + } + } - const { data }: ScrapeResult = await scrapeIt( - `https://bandcamp.com/${BANDCAMP_USERNAME}`, - { - collectionItems: { - listItem: '.collection-item-container', - data: { - url: { - selector: '.collection-title-details > a.item-link', - attr: 'href' - }, - artwork: { - selector: 'div.collection-item-art-container a img', - attr: 'src' - }, - title: { - selector: 'span.item-link-alt > div.collection-item-title' - }, - artist: { - selector: 'span.item-link-alt > div.collection-item-artist' - } - } - } - } - ); + const { data }: ScrapeResult = await scrapeIt(`https://bandcamp.com/${BANDCAMP_USERNAME}`, { + collectionItems: { + listItem: '.collection-item-container', + data: { + url: { + selector: '.collection-title-details > a.item-link', + attr: 'href', + }, + artwork: { + selector: 'div.collection-item-art-container a img', + attr: 'src', + }, + title: { + selector: 'span.item-link-alt > div.collection-item-title', + }, + artist: { + selector: 'span.item-link-alt > div.collection-item-artist', + }, + }, + }, + }); - const albums: Album[] = data?.collectionItems || []; + const albums: Album[] = data?.collectionItems || []; - if (albums && albums?.length > 0) { - if (USE_REDIS_CACHE === 'true') { - redis.set('bandcampAlbums', JSON.stringify(albums), 'EX', 43200); - } - return albums; - } else { - return []; - } - } catch (error) { - console.error(error); - return []; - } + if (albums && albums?.length > 0) { + if (USE_REDIS_CACHE === 'true') { + // Store in Redis for 12 hours. + redis.set('bandcampAlbums', JSON.stringify(albums), 'EX', 43200); + // Reflect the cache TTL on the returned array as a hint to clients. + Object.defineProperty(albums, 'cacheControl', { + value: 'max-age=43200', + enumerable: false, + }); + } + return albums as Album[] & { cacheControl?: string }; + } + return [] as Album[] & { cacheControl?: string }; + } catch (error) { + console.error(error); + return [] as Album[] & { cacheControl?: string }; + } } diff --git a/src/routes/api/articles/+server.ts b/src/routes/api/articles/+server.ts index 55550bb..aa4e3e6 100644 --- a/src/routes/api/articles/+server.ts +++ b/src/routes/api/articles/+server.ts @@ -1,6 +1,6 @@ import { json, error } from '@sveltejs/kit'; import { PAGE_SIZE } from '$env/static/private'; -import { fetchArticlesApi } from '$lib/api'; +import { fetchArticlesApi } from '$lib/services/articlesApi'; import type { ArticlePageLoad } from '@/lib/types/article.js'; export async function GET({ setHeaders, url }) { diff --git a/test-results/.last-run.json b/test-results/.last-run.json new file mode 100644 index 0000000..cbcc1fb --- /dev/null +++ b/test-results/.last-run.json @@ -0,0 +1,4 @@ +{ + "status": "passed", + "failedTests": [] +} \ No newline at end of file