diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..09f7af7 --- /dev/null +++ b/.env.example @@ -0,0 +1,13 @@ +WALLABAG_MAX_ARTICLES=30 +WALLABAG_CLIENT_ID= +WALLABAG_CLIENT_SECRET= +WALLABAG_USERNAME= +WALLABAG_PASSWORD= +WALLABAG_URL="https://app.wallabag.it" +PUBLIC_SITE_URL= +PUBLIC_UMAMI_DO_NOT_TRACK=true +PUBLIC_UMAMI_URL= +PUBLIC_UMAMI_ID= +PAGE_SIZE=6 +USE_REDIS_CACHE=true +REDIS_URI=redis://{username}:{password}@{redisURL}:{redisPort} \ No newline at end of file diff --git a/package.json b/package.json index 98502ca..b56422f 100644 --- a/package.json +++ b/package.json @@ -53,5 +53,8 @@ "vite-imagetools": "^4.0.18", "vitest": "^0.25.3" }, - "type": "module" + "type": "module", + "dependencies": { + "ioredis": "^5.3.1" + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ed38893..acebe6f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,7 @@ specifiers: eslint-config-prettier: ^8.5.0 eslint-plugin-svelte3: ^4.0.0 iconify-icon: ^1.0.5 + ioredis: ^5.3.1 just-intersect: ^4.3.0 postcss: ^8.4.21 postcss-import: ^15.1.0 @@ -40,6 +41,9 @@ specifiers: vite-imagetools: ^4.0.18 vitest: ^0.25.3 +dependencies: + ioredis: 5.3.1 + devDependencies: '@iconify-icons/material-symbols': 1.2.27 '@iconify-icons/radix-icons': 1.2.8 @@ -581,6 +585,10 @@ packages: resolution: {integrity: sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==} dev: true + /@ioredis/commands/1.2.0: + resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==} + dev: false + /@jridgewell/resolve-uri/3.1.0: resolution: {integrity: sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==} engines: {node: '>=6.0.0'} @@ -1330,6 +1338,11 @@ packages: engines: {node: '>=10'} dev: true + /cluster-key-slot/1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + dev: false + /color-convert/2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -1499,7 +1512,6 @@ packages: optional: true dependencies: ms: 2.1.2 - dev: true /decompress-response/6.0.0: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} @@ -1545,6 +1557,11 @@ packages: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} dev: true + /denque/2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + dev: false + /detect-indent/6.1.0: resolution: {integrity: sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==} engines: {node: '>=8'} @@ -2245,6 +2262,23 @@ packages: resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} dev: true + /ioredis/5.3.1: + resolution: {integrity: sha512-C+IBcMysM6v52pTLItYMeV4Hz7uriGtoJdz7SSBDX6u+zwSYGirLdQh3L7t/OItWITcw3gTFMjJReYUwS4zihg==} + engines: {node: '>=12.22.0'} + dependencies: + '@ioredis/commands': 1.2.0 + cluster-key-slot: 1.1.2 + debug: 4.3.4 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + /is-arrayish/0.3.2: resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==} dev: true @@ -2425,6 +2459,14 @@ packages: p-locate: 5.0.0 dev: true + /lodash.defaults/4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + dev: false + + /lodash.isarguments/3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + dev: false + /lodash.merge/4.6.2: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} dev: true @@ -2574,7 +2616,6 @@ packages: /ms/2.1.2: resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} - dev: true /nanoid/3.3.4: resolution: {integrity: sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==} @@ -3320,6 +3361,18 @@ packages: picomatch: 2.3.1 dev: true + /redis-errors/1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + dev: false + + /redis-parser/3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + dependencies: + redis-errors: 1.2.0 + dev: false + /regex-escape/3.4.10: resolution: {integrity: sha512-qEqf7uzW+iYcKNLMDFnMkghhQBnGdivT6KqVQyKsyjSWnoFyooXVnxrw9dtv3AFLnD6VBGXxtZGAQNFGFTnCqA==} dev: true @@ -3612,6 +3665,10 @@ packages: dev: true optional: true + /standard-as-callback/2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + dev: false + /streamsearch/1.1.0: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} diff --git a/src/lib/server/redis.ts b/src/lib/server/redis.ts new file mode 100644 index 0000000..df5e122 --- /dev/null +++ b/src/lib/server/redis.ts @@ -0,0 +1,4 @@ +import { Redis } from 'ioredis'; +import { REDIS_URI } from '$env/static/private'; + +export const redis = new Redis(REDIS_URI); diff --git a/src/routes/api.ts b/src/routes/api.ts index df9fa4a..cc9ee9e 100644 --- a/src/routes/api.ts +++ b/src/routes/api.ts @@ -5,13 +5,15 @@ import { WALLABAG_PASSWORD, WALLABAG_URL, WALLABAG_MAX_ARTICLES, - PAGE_SIZE + PAGE_SIZE, + USE_REDIS_CACHE } from '$env/static/private'; import intersect from 'just-intersect'; import type { Article, WallabagArticle } from '$root/lib/types/article'; import { ArticleTag } from '$root/lib/types/articleTag'; import type { PageQuery } from '$root/lib/types/pageQuery'; import { URLSearchParams } from 'url'; +import { redis } from '$root/lib/server/redis'; const base: string = WALLABAG_URL; @@ -21,7 +23,27 @@ export async function fetchArticlesApi( queryParams: Record, data?: Record ) { - let lastFetched: Date | null = null; + const pageQuery: PageQuery = { + sort: 'updated', + perPage: +PAGE_SIZE, + since: 0, + page: +queryParams?.page || 1, + tags: 'programming', + content: 'metadata' + }; + const entriesQueryParams = new URLSearchParams(pageQuery); + console.log(`Entries params: ${entriesQueryParams}`); + if (USE_REDIS_CACHE) { + const cached = await redis.get(entriesQueryParams.toString()); + + if (cached) { + const response = JSON.parse(cached); + console.log('Cache hit!'); + const ttl = await redis.ttl(entriesQueryParams.toString()); + + return { ...response, cacheControl: `max-age=${ttl}` }; + } + } const authBody = { grant_type: 'password', @@ -39,24 +61,6 @@ export async function fetchArticlesApi( const auth = await authResponse.json(); - const pageQuery: PageQuery = { - sort: 'updated', - perPage: +PAGE_SIZE, - since: 0, - page: +queryParams?.page || 1, - tags: 'programming', - content: 'metadata' - }; - const entriesQueryParams = new URLSearchParams(pageQuery); - console.log(`Entries params: ${entriesQueryParams}`); - - if (lastFetched) { - pageQuery.since = Math.round(lastFetched / 1000); - } - - lastFetched = new Date(); - - const nbEntries = 0; const pageResponse = await fetch(`${WALLABAG_URL}/api/entries.json?${entriesQueryParams}`, { method: 'GET', headers: { @@ -99,20 +103,7 @@ export async function fetchArticlesApi( } }); - // if (!entries._links.next) { - // return; - // } - // console.log(`Links next ${JSON.stringify(entries._links.next)}`); - // const response = await fetch(entries._links.next.href, { - // method: 'GET', - // headers: { - // Authorization: `Bearer ${auth.access_token}` - // } - // }); - // entries = await response.json(); - // } while (entries._links.next); - - return { + const responseData = { articles, currentPage: page, totalPages: pages, @@ -120,4 +111,10 @@ export async function fetchArticlesApi( totalArticles: total, cacheControl }; + + if (USE_REDIS_CACHE) { + redis.set(entriesQueryParams.toString(), JSON.stringify(responseData), 'EX', 43200); + } + + return responseData; } diff --git a/src/routes/api/articles/+server.ts b/src/routes/api/articles/+server.ts index 4b22943..d5dcdee 100644 --- a/src/routes/api/articles/+server.ts +++ b/src/routes/api/articles/+server.ts @@ -21,15 +21,6 @@ export const GET: RequestHandler = async ({ url, setHeaders }: RequestEvent) => } } - // const articlesResponse = response.articles; - // console.log(`Found articles ${articlesResponse?.articles?.length}`); - // const articles = []; - - // for (const article of articlesResponse) { - // const { tags, title, url, hashed_url, reading_time, preview_picture } = article; - // articles.push({ tags, title, url, hashed_url, reading_time, preview_picture }); - // } - return json(response); } } catch (error) {